diff --git a/backend/alembic/versions/a1b2c3d4e5f6_onboarding_help_thread_ts.py b/backend/alembic/versions/a1b2c3d4e5f6_onboarding_help_thread_ts.py new file mode 100644 index 00000000000..cbf79348db7 --- /dev/null +++ b/backend/alembic/versions/a1b2c3d4e5f6_onboarding_help_thread_ts.py @@ -0,0 +1,29 @@ +"""onboarding_request.help_thread_ts + +Revision ID: a1b2c3d4e5f6 +Revises: c40f5a073fc2 +Create Date: 2026-07-18 00:00:00.000000 + +Adds help_thread_ts: the Slack ts of the customer-facing root message posted on +submit, so lifecycle updates (approved / complete / failed) can be posted as +replies in that thread. Additive nullable column — no impact on existing rows. +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = "a1b2c3d4e5f6" +down_revision = "c40f5a073fc2" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "onboarding_request", + sa.Column("help_thread_ts", sa.String(), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("onboarding_request", "help_thread_ts") diff --git a/backend/alembic/versions/c40f5a073fc2_onboarding_request.py b/backend/alembic/versions/c40f5a073fc2_onboarding_request.py new file mode 100644 index 00000000000..c67cf111b27 --- /dev/null +++ b/backend/alembic/versions/c40f5a073fc2_onboarding_request.py @@ -0,0 +1,75 @@ +"""onboarding_request + +Revision ID: c40f5a073fc2 +Revises: 31f30b318163 +Create Date: 2026-07-17 00:00:00.000000 + +Adds the onboarding_request table backing the self-serve team-onboarding flow +(form -> admin approval -> auto-provision + scrape). All columns additive/new +table, so no impact on existing data. +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "c40f5a073fc2" +down_revision = "31f30b318163" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "onboarding_request", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "requester_id", + postgresql.UUID(as_uuid=True), + sa.ForeignKey("user.id"), + nullable=True, + ), + sa.Column("requester_email", sa.String(), nullable=False), + sa.Column("status", sa.String(), nullable=False, server_default="pending"), + sa.Column("payload", postgresql.JSONB(), nullable=False), + sa.Column( + "approver_id", + postgresql.UUID(as_uuid=True), + sa.ForeignKey("user.id"), + nullable=True, + ), + sa.Column("decision_reason", sa.Text(), nullable=True), + sa.Column("error_msg", sa.Text(), nullable=True), + sa.Column( + "persona_id", sa.Integer(), sa.ForeignKey("persona.id"), nullable=True + ), + sa.Column( + "document_set_id", + sa.Integer(), + sa.ForeignKey("document_set.id"), + nullable=True, + ), + sa.Column( + "slack_bot_config_id", + sa.Integer(), + sa.ForeignKey("slack_bot_config.id"), + nullable=True, + ), + sa.Column("cc_pair_ids", postgresql.JSONB(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + ) + + +def downgrade() -> None: + op.drop_table("onboarding_request") diff --git a/backend/danswer/background/update.py b/backend/danswer/background/update.py index 1c7790ac045..773deb03a8a 100755 --- a/backend/danswer/background/update.py +++ b/backend/danswer/background/update.py @@ -38,6 +38,7 @@ from danswer.db.models import IndexingStatus from danswer.db.models import IndexModelStatus from danswer.db.swap_index import check_index_swap +from danswer.onboarding.provision import finalize_ready_onboarding_requests from danswer.search.search_nlp_models import warm_up_encoders from danswer.utils.logger import setup_logger from danswer.utils.variable_functionality import global_version @@ -569,6 +570,11 @@ def update_loop(delay: int = 10, num_workers: int = NUM_INDEXING_WORKERS) -> Non client_secondary = SimpleJobClient(n_workers=num_workers) existing_jobs: dict[int, Future | SimpleJob] = {} + # Onboarding requests are finalized (doc set + assistant + Slack config) once + # all their sources finish scraping. Sweep for ready ones about once a minute + # rather than every scheduler tick. + last_onboarding_sweep = 0.0 + onboarding_sweep_interval = 60.0 while True: start = time.time() @@ -594,6 +600,17 @@ def update_loop(delay: int = 10, num_workers: int = NUM_INDEXING_WORKERS) -> Non ) except Exception as e: logger.exception(f"Failed to run update due to {e}") + + # Onboarding finalization sweep (~once a minute), isolated so a failure + # here never disrupts indexing scheduling. + if start - last_onboarding_sweep >= onboarding_sweep_interval: + last_onboarding_sweep = start + try: + with Session(get_sqlalchemy_engine()) as db_session: + finalize_ready_onboarding_requests(db_session) + except Exception: + logger.exception("Onboarding finalization sweep failed") + sleep_time = delay - (time.time() - start) if sleep_time > 0: time.sleep(sleep_time) diff --git a/backend/danswer/connectors/slack/connector.py b/backend/danswer/connectors/slack/connector.py index 76d4dcdea80..d1348b9e7c2 100644 --- a/backend/danswer/connectors/slack/connector.py +++ b/backend/danswer/connectors/slack/connector.py @@ -27,6 +27,7 @@ from danswer.connectors.slack.utils import make_slack_api_call_logged from danswer.connectors.slack.utils import make_slack_api_call_paginated from danswer.connectors.slack.utils import make_slack_api_rate_limited +from danswer.connectors.slack.utils import resolve_workspace_subdomain from danswer.connectors.slack.utils import SlackTextCleaner from danswer.utils.logger import setup_logger @@ -323,6 +324,11 @@ def get_all_docs( client=client, channel=channel, oldest=oldest, latest=latest ) + # Resolve the channel's true workspace subdomain (Grid-safe) lazily on + # the first message, then reuse it for every thread in the channel — one + # chat.getPermalink call per channel rather than per message. + channel_workspace: str | None = None + seen_thread_ts: set[str] = set() for message_batch in channel_message_batches: for message in message_batch: @@ -344,9 +350,16 @@ def get_all_docs( filtered_thread = [message] if filtered_thread: + if channel_workspace is None: + channel_workspace = resolve_workspace_subdomain( + client=client, + channel_id=channel["id"], + message_ts=filtered_thread[0]["ts"], + fallback=workspace, + ) channel_docs += 1 yield thread_to_doc( - workspace=workspace, + workspace=channel_workspace, channel=channel, thread=filtered_thread, slack_cleaner=slack_cleaner, diff --git a/backend/danswer/connectors/slack/utils.py b/backend/danswer/connectors/slack/utils.py index 48a591f2aca..7f60dea6866 100644 --- a/backend/danswer/connectors/slack/utils.py +++ b/backend/danswer/connectors/slack/utils.py @@ -5,6 +5,7 @@ from functools import wraps from typing import Any from typing import cast +from urllib.parse import urlparse from slack_sdk import WebClient from slack_sdk.errors import SlackApiError @@ -18,6 +19,23 @@ # number of messages we request per page when fetching paginated slack messages _SLACK_LIMIT = 900 +# Mis-stored Slack workspace subdomains -> their correct URL subdomain. +# Historically some connectors were configured with the workspace *display name* +# ("Product") instead of the URL subdomain ("uipath-product"), so their stored +# permalinks point at the dead host `product.slack.com`. Keys are matched case- +# and whitespace-insensitively, so this covers both "Product" and "Product ". +# This is an explicit allow-map on purpose: only listed subdomains are rewritten, +# so genuinely different workspaces (uipath-customer-ops, uipath-marketing, ...) +# and links already on a correct host are left untouched. Add an entry here if a +# new mis-configured workspace surfaces. +_SLACK_SUBDOMAIN_FIXES = { + "product": "uipath-product", +} + +# Captures the subdomain in the `//.slack.com` portion of a permalink. +# `[^/]*?` is non-greedy and tolerates a stray space ("Product .slack.com"). +_SLACK_HOST_RE = re.compile(r"(//)([^/]*?)(\.slack\.com)", re.IGNORECASE) + def get_message_link( event: dict[str, Any], workspace: str, channel_id: str | None = None @@ -34,6 +52,66 @@ def get_message_link( ) +def normalize_slack_link(url: str) -> str: + """Rewrite a mis-stored Slack workspace subdomain to the canonical one. + + Existing indexed docs carry links built from a mis-configured workspace + ("Product" -> dead `product.slack.com`). Retrieval reads every citation + link back through here, so chat / search / Slack-bot citations all resolve + to the real workspace without re-indexing or a data migration. Only the + subdomains in `_SLACK_SUBDOMAIN_FIXES` are rewritten; a link on the canonical + host, or on a genuinely different workspace, is returned unchanged.""" + if not url or ".slack.com" not in url.lower(): + return url + + def _fix(match: "re.Match[str]") -> str: + subdomain = match.group(2).strip().lower() + canonical = _SLACK_SUBDOMAIN_FIXES.get(subdomain) + if canonical is not None: + return f"{match.group(1)}{canonical}{match.group(3)}" + return match.group(0) + + return _SLACK_HOST_RE.sub(_fix, url, count=1) + + +def resolve_workspace_subdomain( + client: WebClient, channel_id: str, message_ts: str, fallback: str +) -> str: + """Ask Slack for the authoritative workspace subdomain of a channel. + + `chat.getPermalink` returns the real permalink for a message, with the + correct `.slack.com` host even under Enterprise Grid (where a single + bot can see channels that live in different workspaces, each with its own + URL). We resolve this ONCE per channel and reuse it, so the connector no + longer depends on a hand-typed `workspace` config being right. Falls back to + `fallback` (the configured workspace) if the call fails.""" + try: + # Same rate-limit + logging wrapping as every other Slack call in the + # connector, so a 429 retries (with Retry-After) instead of falling back. + resp = make_slack_api_rate_limited( + make_slack_api_call_logged(client.chat_getPermalink) + )(channel=channel_id, message_ts=message_ts) + permalink = cast(str, resp.get("permalink") or "") + host = urlparse(permalink).netloc.lower() + if host.endswith(".slack.com"): + subdomain = host[: -len(".slack.com")] + if subdomain: + return subdomain + except SlackApiError as e: + logger.warning( + "chat.getPermalink failed for channel %s (%s); " + "falling back to configured workspace '%s'", + channel_id, + e.response.get("error"), + fallback, + ) + except Exception as e: + logger.warning( + "could not resolve workspace subdomain for channel %s: %s", channel_id, e + ) + return fallback + + def make_slack_api_call_logged( call: Callable[..., SlackResponse], ) -> Callable[..., SlackResponse]: diff --git a/backend/danswer/connectors/web/connector.py b/backend/danswer/connectors/web/connector.py index f66346a9d15..50b34dac4b8 100644 --- a/backend/danswer/connectors/web/connector.py +++ b/backend/danswer/connectors/web/connector.py @@ -1,5 +1,6 @@ import io import ipaddress +import os import random import re import socket @@ -161,7 +162,13 @@ def get_internal_links( def start_playwright() -> Tuple[Playwright, BrowserContext]: playwright = sync_playwright().start() - browser = playwright.chromium.launch(headless=True) + # Optionally launch an installed browser (e.g. system Chrome) instead of the + # bundled Chromium. Prod leaves this unset and uses the pinned Chromium in + # the Linux image; it exists only so local dev can fall back to a working + # browser when the pinned build is incompatible with the host OS (the + # chromium-1097 build for playwright 1.41.2 SIGSEGVs on recent macOS). + browser_channel = os.environ.get("WEB_CONNECTOR_BROWSER_CHANNEL") or None + browser = playwright.chromium.launch(headless=True, channel=browser_channel) context = browser.new_context(user_agent=DEFAULT_USER_AGENT) @@ -314,7 +321,9 @@ def _uipath_product_prefix(path: str) -> str: return "/" + "/".join(segs[:cut]) -def get_uipath_docs_version_base_urls(base_url: str, max_versions: int = 2) -> list[str]: +def get_uipath_docs_version_base_urls( + base_url: str, max_versions: int = 2 +) -> list[str]: """Expand a docs.uipath.com product URL to the base URLs of its latest N concrete versions. diff --git a/backend/danswer/db/models.py b/backend/danswer/db/models.py index ff01513bf14..7b05bf21bb1 100644 --- a/backend/danswer/db/models.py +++ b/backend/danswer/db/models.py @@ -1716,3 +1716,77 @@ class ChatReferral(Base): created_at: Mapped[datetime.datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() ) + + +class OnboardingStatus(str, PyEnum): + """Lifecycle of a team's self-serve Darwin onboarding request.""" + + PENDING = "pending" # submitted, awaiting admin approval + REJECTED = "rejected" # admin declined + CANCELLED = "cancelled" # withdrawn by the requester while still pending + PROVISIONING = "provisioning" # approved; creating connectors/persona/config + INDEXING = "indexing" # resources created; sources scraping + COMPLETE = "complete" # all sources indexed successfully + FAILED = "failed" # provisioning or indexing failed + + +class OnboardingRequest(Base): + """A self-serve request to onboard a team/channel onto Darwin. Anyone may + submit; only an admin may approve. On approval the provisioning orchestrator + (onboarding/provision.py) creates the connectors, document set, assistant, and + Slack bot config, then records their ids here so the requester can monitor the + per-source scrape status.""" + + __tablename__ = "onboarding_request" + + id: Mapped[int] = mapped_column(primary_key=True) + # Who submitted (denormalize email so it survives user deletion / is display-ready). + requester_id: Mapped[UUID | None] = mapped_column( + ForeignKey("user.id"), nullable=True + ) + requester_email: Mapped[str] = mapped_column(String, nullable=False) + # Stored as the enum VALUE string (e.g. "pending"), NOT via SA Enum() — the + # repo's Enum(native_enum=False) stores the NAME (uppercase), which is a known + # footgun. Plain String of `.value` keeps it consistent with the migration. + status: Mapped[str] = mapped_column( + String, nullable=False, default=OnboardingStatus.PENDING.value + ) + # The full validated form (channel, ordered sources, prompt, SME/oncall/jira + # options, docs cloud/onprem roots, etc.) — JSONB for flexibility. + payload: Mapped[dict] = mapped_column(postgresql.JSONB(), nullable=False) + # Admin who approved/rejected + optional reason. + approver_id: Mapped[UUID | None] = mapped_column( + ForeignKey("user.id"), nullable=True + ) + decision_reason: Mapped[str | None] = mapped_column(Text, nullable=True) + # Free-text error surfaced when status == FAILED. + error_msg: Mapped[str | None] = mapped_column(Text, nullable=True) + # Provisioned resource ids (populated on approval; drive the status monitor). + persona_id: Mapped[int | None] = mapped_column( + ForeignKey("persona.id"), nullable=True + ) + document_set_id: Mapped[int | None] = mapped_column( + ForeignKey("document_set.id"), nullable=True + ) + slack_bot_config_id: Mapped[int | None] = mapped_column( + ForeignKey("slack_bot_config.id"), nullable=True + ) + # cc_pair ids created for this onboarding (list[int]); the monitor reads their + # indexing status. JSONB list rather than a join table — always used together. + cc_pair_ids: Mapped[list[int] | None] = mapped_column( + postgresql.JSONB(), nullable=True + ) + # Slack message ts of the customer-facing root message posted to #help-darwin + # on submit. Subsequent lifecycle updates (approved / complete / failed) are + # posted as replies in this thread (thread_ts), so the requester tracks their + # request in one quiet thread. Null if the root post failed/was skipped. + help_thread_ts: Mapped[str | None] = mapped_column(String, nullable=True) + created_at: Mapped[datetime.datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime.datetime] = mapped_column( + DateTime(timezone=True), + nullable=False, + server_default=func.now(), + onupdate=func.now(), + ) diff --git a/backend/danswer/db/onboarding.py b/backend/danswer/db/onboarding.py new file mode 100644 index 00000000000..2ee819fc256 --- /dev/null +++ b/backend/danswer/db/onboarding.py @@ -0,0 +1,123 @@ +"""DB helpers for the self-serve Darwin onboarding flow (see OnboardingRequest). + +Status is stored as the enum VALUE string; always write/compare with +OnboardingStatus(...).value to stay consistent with the column + migration.""" +from uuid import UUID + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from danswer.db.models import OnboardingRequest +from danswer.db.models import OnboardingStatus + + +def create_onboarding_request( + db_session: Session, + requester_id: UUID | None, + requester_email: str, + payload: dict, +) -> OnboardingRequest: + request = OnboardingRequest( + requester_id=requester_id, + requester_email=requester_email, + payload=payload, + status=OnboardingStatus.PENDING.value, + ) + db_session.add(request) + db_session.commit() + return request + + +def get_onboarding_request( + db_session: Session, request_id: int +) -> OnboardingRequest | None: + return db_session.get(OnboardingRequest, request_id) + + +def list_onboarding_requests( + db_session: Session, status: OnboardingStatus | None = None +) -> list[OnboardingRequest]: + """All requests (admin view), newest first; optionally filtered by status.""" + stmt = select(OnboardingRequest).order_by(OnboardingRequest.created_at.desc()) + if status is not None: + stmt = stmt.where(OnboardingRequest.status == status.value) + return list(db_session.execute(stmt).scalars().all()) + + +def list_onboarding_requests_for_user( + db_session: Session, requester_id: UUID +) -> list[OnboardingRequest]: + """A requester's own submissions, newest first.""" + stmt = ( + select(OnboardingRequest) + .where(OnboardingRequest.requester_id == requester_id) + .order_by(OnboardingRequest.created_at.desc()) + ) + return list(db_session.execute(stmt).scalars().all()) + + +def update_onboarding_status( + db_session: Session, + request: OnboardingRequest, + status: OnboardingStatus, + *, + approver_id: UUID | None = None, + decision_reason: str | None = None, + error_msg: str | None = None, + commit: bool = True, +) -> OnboardingRequest: + request.status = status.value + if approver_id is not None: + request.approver_id = approver_id + if decision_reason is not None: + request.decision_reason = decision_reason + # error_msg is cleared on a non-failed transition, set on failure. + request.error_msg = error_msg if status == OnboardingStatus.FAILED else None + if commit: + db_session.commit() + return request + + +def update_onboarding_payload( + db_session: Session, request: OnboardingRequest, payload: dict +) -> OnboardingRequest: + """Replace a request's payload (admin edits a pending request before approving). + JSONB is only re-persisted on reassignment, so set a fresh dict.""" + request.payload = dict(payload) + db_session.commit() + return request + + +def set_provisioned_ids( + db_session: Session, + request: OnboardingRequest, + *, + persona_id: int | None = None, + document_set_id: int | None = None, + slack_bot_config_id: int | None = None, + cc_pair_ids: list[int] | None = None, + commit: bool = True, +) -> OnboardingRequest: + """Record the resources provisioning created, so the status monitor can map + the request to its cc_pairs / assistant.""" + if persona_id is not None: + request.persona_id = persona_id + if document_set_id is not None: + request.document_set_id = document_set_id + if slack_bot_config_id is not None: + request.slack_bot_config_id = slack_bot_config_id + if cc_pair_ids is not None: + request.cc_pair_ids = cc_pair_ids + if commit: + db_session.commit() + return request + + +def set_help_thread_ts( + db_session: Session, request: OnboardingRequest, thread_ts: str +) -> OnboardingRequest: + """Persist the Slack ts of the customer-facing root message so later lifecycle + updates can be posted as replies in that thread.""" + request.help_thread_ts = thread_ts + db_session.commit() + return request diff --git a/backend/danswer/document_index/vespa/index.py b/backend/danswer/document_index/vespa/index.py index a07b292ca94..ac868bfab36 100644 --- a/backend/danswer/document_index/vespa/index.py +++ b/backend/danswer/document_index/vespa/index.py @@ -59,6 +59,7 @@ from danswer.connectors.cross_connector_utils.miscellaneous_utils import ( get_experts_stores_representations, ) +from danswer.connectors.slack.utils import normalize_slack_link from danswer.document_index.document_index_utils import get_uuid_from_chunk from danswer.document_index.interfaces import DocumentIndex from danswer.document_index.interfaces import DocumentInsertionRecord @@ -681,8 +682,12 @@ def _vespa_hit_to_inference_chunk(hit: dict[str, Any]) -> InferenceChunk: source_links_dict_unprocessed = ( json.loads(source_links) if isinstance(source_links, str) else source_links ) + # Slack: historical docs were indexed with a mis-configured workspace, so + # their permalinks point at a dead host. Rewrite to the canonical workspace + # at read time so every citation (chat / search / Slack bot) resolves. + is_slack = str(fields.get(SOURCE_TYPE, "")).lower() == "slack" source_links_dict = { - int(k): v + int(k): (normalize_slack_link(v) if is_slack else v) for k, v in cast(dict[str, str], source_links_dict_unprocessed).items() } diff --git a/backend/danswer/main.py b/backend/danswer/main.py index 879d1edb0dd..d0ab43109db 100644 --- a/backend/danswer/main.py +++ b/backend/danswer/main.py @@ -63,6 +63,10 @@ from danswer.server.documents.document import router as document_router from danswer.server.features.document_set.api import router as document_set_router from danswer.server.features.folder.api import router as folder_router +from danswer.server.features.onboarding.api import ( + admin_router as admin_onboarding_router, +) +from danswer.server.features.onboarding.api import basic_router as onboarding_router from danswer.server.features.persona.api import admin_router as admin_persona_router from danswer.server.features.persona.api import basic_router as persona_router from danswer.server.features.prompt.api import basic_router as prompt_router @@ -280,6 +284,8 @@ def get_application() -> FastAPI: ) include_router_with_global_prefix_prepended(application, persona_router) include_router_with_global_prefix_prepended(application, admin_persona_router) + include_router_with_global_prefix_prepended(application, onboarding_router) + include_router_with_global_prefix_prepended(application, admin_onboarding_router) include_router_with_global_prefix_prepended(application, prompt_router) include_router_with_global_prefix_prepended(application, tool_router) include_router_with_global_prefix_prepended(application, admin_tool_router) diff --git a/backend/danswer/onboarding/__init__.py b/backend/danswer/onboarding/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/backend/danswer/onboarding/notify.py b/backend/danswer/onboarding/notify.py new file mode 100644 index 00000000000..c59d2cdb931 --- /dev/null +++ b/backend/danswer/onboarding/notify.py @@ -0,0 +1,207 @@ +"""Slack notifications for the onboarding workflow. + +Best-effort: a Slack failure must never break the request itself, so every send +is wrapped and only logged on error.""" +import os +import re + +from slack_sdk import WebClient + +from danswer.configs.app_configs import WEB_DOMAIN +from danswer.danswerbot.slack.tokens import fetch_tokens +from danswer.db.models import OnboardingRequest +from danswer.utils.logger import setup_logger + +logger = setup_logger() + +_ARCHIVES_RE = re.compile(r"/archives/(C[A-Z0-9]+)") + + +def _resolve_channel(value: str) -> str: + """Accept a channel id, a bare name, or a pasted channel link (id parsed out).""" + m = _ARCHIVES_RE.search(value or "") + return m.group(1) if m else (value or "").strip() + + +# INTERNAL ops channel — admin review link + real error details. Default +# #darwin-devs (C07B2V8E99S), a PRIVATE cross-Grid channel addressed by id; the +# bot is a member. Override via env/configmap (id, name, or channel link). +ONBOARDING_NOTIFY_CHANNEL = _resolve_channel( + os.environ.get("ONBOARDING_NOTIFY_CHANNEL") or "C07B2V8E99S" +) + +# CUSTOMER-FACING channel — the requester is @mentioned on submit and every +# lifecycle update is posted as a *reply in that one thread* (never new top-level +# messages, so a large channel isn't spammed). Defaults to #darwin-devs for +# testing; set ONBOARDING_CLIENT_CHANNEL to #help-darwin (C077AFEGCKZ) in prod +# once the bot has been added there. +ONBOARDING_CLIENT_CHANNEL = _resolve_channel( + os.environ.get("ONBOARDING_CLIENT_CHANNEL") or "C07B2V8E99S" +) + + +def _post(text: str, request_id: int | None = None) -> None: + """Post to the ops channel. Best-effort — never raises.""" + if not ONBOARDING_NOTIFY_CHANNEL: + return + try: + client = WebClient(token=fetch_tokens().bot_token) + client.chat_postMessage( + channel=ONBOARDING_NOTIFY_CHANNEL, text=text, unfurl_links=False + ) + except Exception as e: + logger.warning( + "failed to notify '%s' of onboarding request %s: %s", + ONBOARDING_NOTIFY_CHANNEL, + request_id, + e, + ) + + +def _team_and_channel(request: OnboardingRequest) -> tuple[str, str]: + payload = request.payload or {} + team = payload.get("team_name") or "?" + channel = (payload.get("channel") or {}).get("channel_name") or "?" + return team, channel + + +def notify_onboarding_submitted(request: OnboardingRequest) -> None: + """A new request was submitted and needs admin review.""" + team, channel = _team_and_channel(request) + _post( + f":inbox_tray: *New Darwin onboarding request* from " + f"*{request.requester_email}*\n" + f"• Team: *{team}* → #{channel}\n" + f"Review & approve: {WEB_DOMAIN}/admin/onboarding/{request.id}", + request.id, + ) + + +def notify_onboarding_complete(request: OnboardingRequest) -> None: + """All sources scraped, assistant wired up, Darwin live in the channel.""" + team, channel = _team_and_channel(request) + _post( + f":white_check_mark: *Darwin is now live in #{channel}* for *{team}* — " + f"all sources scraped, the assistant is wired to the new document set.\n" + f"Status: {WEB_DOMAIN}/admin/onboarding", + request.id, + ) + + +def notify_onboarding_failed(request: OnboardingRequest, detail: str) -> None: + """One or more sources failed to scrape (or provisioning errored).""" + team, channel = _team_and_channel(request) + _post( + f":x: *Darwin onboarding failed for {team}* (#{channel}) — {detail}\n" + f"Details: {WEB_DOMAIN}/admin/onboarding", + request.id, + ) + + +# --- customer-facing thread (#help-darwin) ---------------------------------- + + +def _mention(client: WebClient, email: str | None) -> str: + """Resolve requester email -> Slack @mention; fall back to the email text + (e.g. external address not in the workspace). Never raises.""" + if not email: + return "there" + try: + uid = client.users_lookupByEmail(email=email)["user"]["id"] + return f"<@{uid}>" + except Exception: + return email + + +def _ensure_member(client: WebClient, channel_id: str) -> None: + """Best-effort join so the bot can post to a public channel. No-op/ignored + for private channels (already invited) or when the scope is missing.""" + try: + client.conversations_join(channel=channel_id) + except Exception: + pass + + +def notify_client_submitted(request: OnboardingRequest) -> str | None: + """Root customer-facing message on submit: @mention the requester + tracking + link. Returns the message ts to persist so later updates thread under it. + Returns None (and posts nothing further) on any failure — callers must handle + a missing ts by skipping replies, never by posting top-level.""" + if not ONBOARDING_CLIENT_CHANNEL: + return None + team, channel = _team_and_channel(request) + try: + client = WebClient(token=fetch_tokens().bot_token) + _ensure_member(client, ONBOARDING_CLIENT_CHANNEL) + who = _mention(client, request.requester_email) + resp = client.chat_postMessage( + channel=ONBOARDING_CLIENT_CHANNEL, + text=( + f":wave: Hi {who} — your *Darwin onboarding request* for *{team}* " + f"(#{channel}) has been received. Track its status any time here: " + f"{WEB_DOMAIN}/onboarding?view=requests\n" + f"I'll post updates in this thread as it progresses." + ), + unfurl_links=False, + ) + return resp.get("ts") + except Exception as e: + logger.warning( + "failed to post customer onboarding root for request %s: %s", + request.id, + e, + ) + return None + + +def _client_reply(request: OnboardingRequest, text: str) -> None: + """Post a threaded reply under the request's root message. Skips entirely if + there is no stored thread ts — a large customer channel must never receive + stray top-level messages. Best-effort; never raises.""" + ts = getattr(request, "help_thread_ts", None) + if not ts or not ONBOARDING_CLIENT_CHANNEL: + return + try: + client = WebClient(token=fetch_tokens().bot_token) + client.chat_postMessage( + channel=ONBOARDING_CLIENT_CHANNEL, + text=text, + thread_ts=ts, + unfurl_links=False, + ) + except Exception as e: + logger.warning( + "failed to post customer onboarding reply for request %s: %s", + request.id, + e, + ) + + +def notify_client_approved(request: OnboardingRequest) -> None: + """Reply: admin reviewed + approved; indexing has started.""" + _client_reply( + request, + ":white_check_mark: Your request has been reviewed and *approved* — " + "Darwin is now indexing your sources. I'll update this thread once it's " + "live.", + ) + + +def notify_client_complete(request: OnboardingRequest) -> None: + """Reply: all sources indexed, assistant live in the channel.""" + _, channel = _team_and_channel(request) + _client_reply( + request, + f":tada: *Darwin is now live in #{channel}!* All your sources are indexed " + f"and the assistant is ready — just ask it a question in the channel.", + ) + + +def notify_client_failed(request: OnboardingRequest) -> None: + """Reply: a source failed; team notified. No internal error details here.""" + _client_reply( + request, + ":warning: One of your sources hit a snag while indexing. Our team has " + "been notified and is looking into it — nothing needed from you; I'll " + "update this thread with progress.", + ) diff --git a/backend/danswer/onboarding/provision.py b/backend/danswer/onboarding/provision.py new file mode 100644 index 00000000000..bef5f9d4be6 --- /dev/null +++ b/backend/danswer/onboarding/provision.py @@ -0,0 +1,753 @@ +"""Provisioning orchestrator for approved onboarding requests. + +Two phases, so Darwin only goes live in a channel once it actually has the +knowledge: + +Phase 1 — `provision_onboarding` (on admin approval): + per source -> Connector (+ credential) -> connector_credential_pair + -> one high-priority IndexAttempt per cc_pair (so the new sources scrape first) + -> status INDEXING. Redundant child Confluence pages are dropped (parent only); + per-source scrape cadence is set (Slack/Confluence/Jira daily, docs monthly). + +Phase 2 — `finalize_onboarding` (background sweep, once every source is scraped): + DocumentSet (over the cc_pairs) -> Prompt (Orchestrator default, requester- + edited) -> Persona (tied to the document set) -> slack_bot_config (channel -> + persona, SME / oncall / Jira options, prioritized_sources) -> status COMPLETE, + and a #darwin-devs notification. The sweep (`finalize_ready_onboarding_requests`) + also flags source failures; it re-checks FAILED requests too, so fixing + + re-indexing a source lets it complete with nothing to restart. + +Payload contract (assembled + validated by the form): + { + "team_name": str, + "channel": {"channel_id": str, "channel_name": str}, + "response_type": "citations" | "quotes", + "respond_tag_only": bool, + "system_prompt": str, "task_prompt": str, # requester-edited + "sme": {"enabled": bool, "group_name": str}, + "oncall": {"enabled": bool, "schedule": str}, # opsgenie_schedule + "jira": {"enabled": bool, "project_key": str, "issue_type": str, "component": str}, + "sources": [ {"type": "web"|"confluence"|"github"|"slack", + "value": , "label": str}, ... ] # priority order + } +""" +import ipaddress +import socket +from urllib.parse import urlparse + +from sqlalchemy import desc +from sqlalchemy import select +from sqlalchemy.orm import Session + +from danswer.configs.constants import DocumentSource +from danswer.connectors.confluence.connector import extract_confluence_keys_from_url +from danswer.connectors.models import InputType +from danswer.db.connector import create_connector +from danswer.db.connector_credential_pair import add_credential_to_connector +from danswer.db.connector_credential_pair import get_connector_credential_pair +from danswer.db.connector_credential_pair import get_connector_credential_pair_from_id +from danswer.db.document_set import insert_document_set +from danswer.db.embedding_model import get_current_db_embedding_model +from danswer.db.index_attempt import create_index_attempt +from danswer.db.models import ChannelConfig +from danswer.db.models import Connector +from danswer.db.models import Credential +from danswer.db.models import IndexAttempt +from danswer.db.models import IndexingStatus +from danswer.db.models import OnboardingRequest +from danswer.db.models import OnboardingStatus +from danswer.db.models import Prompt +from danswer.db.models import RecencyBiasSetting +from danswer.db.models import SlackBotResponseType +from danswer.db.models import Tool +from danswer.db.models import User +from danswer.db.onboarding import list_onboarding_requests +from danswer.db.onboarding import set_provisioned_ids +from danswer.db.onboarding import update_onboarding_status +from danswer.db.persona import get_persona_by_name +from danswer.db.persona import upsert_persona +from danswer.db.persona import upsert_prompt +from danswer.db.slack_bot_config import insert_slack_bot_config +from danswer.onboarding.notify import notify_client_approved +from danswer.onboarding.notify import notify_client_complete +from danswer.onboarding.notify import notify_client_failed +from danswer.onboarding.notify import notify_onboarding_complete +from danswer.onboarding.notify import notify_onboarding_failed +from danswer.onboarding.validation import _allowed_confluence_hosts +from danswer.onboarding.validation import confluence_page_ancestors +from danswer.onboarding.validation import jira_base_url +from danswer.onboarding.validation import validate_confluence_url +from danswer.onboarding.validation import validate_github_repo +from danswer.onboarding.validation import validate_jira_filter +from danswer.onboarding.validation import validate_slack_channel +from danswer.server.documents.models import ConnectorBase +from danswer.server.features.document_set.models import DocumentSetCreationRequest +from danswer.utils.logger import setup_logger + +logger = setup_logger() + +# Onboarded sources scrape ahead of routine re-indexing (IndexAttempt priority 0-100). +# in_code_tool_id of the built-in document SearchTool (Tool.in_code_tool_id == +# SearchTool.__name__). Kept as a literal to avoid importing the heavy search +# pipeline into this module. +_SEARCH_TOOL_IN_CODE_ID = "SearchTool" + +ONBOARDING_INDEXING_PRIORITY = 80 +DEFAULT_REFRESH_FREQ = 86400 # daily +_MONTHLY_REFRESH_FREQ = 2592000 # 30 days + +# Per-source scrape cadence: Slack / Confluence / Jira change often (daily); docs +# sites change slowly and are large, so monthly. +REFRESH_FREQ_BY_SOURCE = { + "slack": DEFAULT_REFRESH_FREQ, + "confluence": DEFAULT_REFRESH_FREQ, + "jira": DEFAULT_REFRESH_FREQ, + "web": _MONTHLY_REFRESH_FREQ, # docs +} +PUBLIC_CREDENTIAL_ID = ( + 0 # the empty public credential (create_initial_public_credential) +) +_DEFAULT_TEMPLATE_PERSONA = "Orchestrator" + + +def _find_credential_id(db_session: Session, required_key: str) -> int | None: + """Reuse an existing connector's shared credential for auth'd sources + (Confluence/GitHub/Slack) — the requester never supplies tokens.""" + for cred in db_session.execute(select(Credential)).scalars(): + if (cred.credential_json or {}).get(required_key): + return cred.id + return None + + +def _slack_workspace(db_session: Session) -> str | None: + for connector in ( + db_session.execute( + select(Connector).where(Connector.source == DocumentSource.SLACK) + ) + .scalars() + .all() + ): + ws = (connector.connector_specific_config or {}).get("workspace") + if ws: + return ws + return None + + +def _parse_github_repo(url: str) -> tuple[str, str]: + parts = urlparse(url).path.strip("/").split("/") + if len(parts) < 2: + raise ValueError(f"Not a github repo URL: {url}") + return parts[0], parts[1] + + +# Hosts we accept for GitHub sources. The connector authenticates against the +# GitHub API (not the URL host), but pinning the host keeps non-GitHub URLs out. +_ALLOWED_GITHUB_HOSTS = {"github.com", "www.github.com"} + + +def _assert_public_web_host(url: str) -> None: + """SSRF guard for web sources: require an http(s) URL whose host does not + resolve to a private / loopback / link-local / reserved address, so an + onboarding request can't point the crawler at internal infrastructure. + + Fails closed: a URL we can't parse or resolve is rejected rather than + fetched. This is a provision-time check; it doesn't defend against DNS + rebinding at fetch time, but it closes the obvious internal-target vector.""" + parsed = urlparse(url) + if parsed.scheme not in ("http", "https"): + raise ValueError(f"Web source URL must be http(s): {url}") + host = parsed.hostname + if not host: + raise ValueError(f"Web source URL has no host: {url}") + try: + addr_infos = socket.getaddrinfo(host, None) + except OSError as e: + raise ValueError(f"Could not resolve web source host '{host}': {e}") + for info in addr_infos: + ip = ipaddress.ip_address(info[4][0]) + if ( + ip.is_private + or ip.is_loopback + or ip.is_link_local + or ip.is_reserved + or ip.is_multicast + or ip.is_unspecified + ): + raise ValueError( + f"Web source host '{host}' resolves to a non-public address ({ip})" + ) + + +def _assert_source_accessible(source: dict, db_session: Session) -> None: + """Fail-fast at provision: confirm the reusable credential can actually reach + the source, so we never create a connector that would silently index nothing. + Reuses the same validators as the submit-time /validate endpoint. Web sources + are public (no credential) and are host-guarded in `_build_connector_base`.""" + stype = source["type"] + value = source["value"] + if stype == "confluence": + result = validate_confluence_url(value, db_session) + elif stype == "github": + result = validate_github_repo(value, db_session) + elif stype == "slack": + result = validate_slack_channel(value) + elif stype == "jira": + result = validate_jira_filter(value, db_session) + else: + return + if not result.valid: + raise ValueError(f"{stype} source not accessible: {result.message}") + + +def _build_connector_base(source: dict, db_session: Session) -> ConnectorBase: + """One onboarding source -> a ConnectorBase (source-specific config).""" + stype = source["type"] + value = source["value"] + label = source.get("label") or value + refresh_freq = REFRESH_FREQ_BY_SOURCE.get(stype, DEFAULT_REFRESH_FREQ) + + if stype == "web": + # SSRF guard: never let a submitted URL point the crawler at internal + # hosts (validation at submit time can be bypassed by a direct API call). + _assert_public_web_host(value) + is_uipath_docs = urlparse(value).netloc == "docs.uipath.com" + config: dict = {"base_url": value, "web_connector_type": "recursive"} + if is_uipath_docs: + # Strip the version from the root URL and crawl the latest few + # concrete versions (no need to list each version URL). + config["uipath_latest_versions"] = True + config["max_versions"] = 3 + return ConnectorBase( + name=f"[onboarding] {label}", + source=DocumentSource.WEB, + input_type=InputType.POLL, + connector_specific_config=config, + refresh_freq=refresh_freq, + prune_freq=None, + disabled=False, + ) + if stype == "confluence": + # Credential-leak / SSRF guard: provisioning creates a connector that + # will authenticate to this host with our stored Confluence token, so + # only allow a host an existing Confluence connector already uses. Re-run + # here (not just at submit) because provisioning is the point the + # credential is actually used, and submit-time validation can be skipped. + host = urlparse(value).netloc.lower() + allowed_hosts = _allowed_confluence_hosts(db_session) + if not allowed_hosts or host not in allowed_hosts: + raise ValueError( + f"Confluence host '{host}' is not allowed — it must match an " + f"existing Confluence connector ({sorted(allowed_hosts)})" + ) + return ConnectorBase( + name=f"[onboarding] {label}", + source=DocumentSource.CONFLUENCE, + input_type=InputType.POLL, + connector_specific_config={"wiki_page_url": value}, + refresh_freq=refresh_freq, + prune_freq=None, + disabled=False, + ) + if stype == "github": + gh_host = (urlparse(value).hostname or "").lower() + if gh_host not in _ALLOWED_GITHUB_HOSTS: + raise ValueError(f"GitHub source URL must be on github.com: {value}") + owner, repo = _parse_github_repo(value) + return ConnectorBase( + name=f"[onboarding] {label}", + source=DocumentSource.GITHUB, + input_type=InputType.POLL, + connector_specific_config={ + "repo_owner": owner, + "repo_name": repo, + "include_prs": True, + "include_issues": True, + }, + refresh_freq=refresh_freq, + prune_freq=None, + disabled=False, + ) + if stype == "slack": + workspace = _slack_workspace(db_session) + if not workspace: + raise ValueError("No existing Slack connector to copy the workspace from") + return ConnectorBase( + name=f"[onboarding] {label}", + source=DocumentSource.SLACK, + input_type=InputType.POLL, + connector_specific_config={ + "workspace": workspace, + "channels": [value.lstrip("#")], + "channel_regex_enabled": False, + }, + refresh_freq=refresh_freq, + prune_freq=None, + disabled=False, + ) + if stype == "jira": + # value is a JQL filter; reuse an existing Jira connector's base URL so + # the requester never supplies the host/credentials. + base = jira_base_url(db_session) + if not base: + raise ValueError("No existing Jira connector to copy the base URL from") + return ConnectorBase( + name=f"[onboarding] {label}", + source=DocumentSource.JIRA, + input_type=InputType.POLL, + connector_specific_config={"jira_base_url": base, "jira_filter": value}, + refresh_freq=refresh_freq, + prune_freq=None, + disabled=False, + ) + raise ValueError(f"Unsupported source type: {stype}") + + +# Which decrypted-credential key identifies a reusable credential per source type. +_CREDENTIAL_KEY_BY_SOURCE = { + "confluence": "confluence_access_token", + "github": "github_access_token", + "slack": "slack_bot_token", + "jira": "jira_api_token", +} + + +def _credential_id_for_source(source_type: str, db_session: Session) -> int: + if source_type == "web": + return PUBLIC_CREDENTIAL_ID + key = _CREDENTIAL_KEY_BY_SOURCE.get(source_type) + if key is None: + return PUBLIC_CREDENTIAL_ID + cred_id = _find_credential_id(db_session, key) + if cred_id is None: + raise ValueError( + f"No existing {source_type} credential to reuse (missing '{key}')" + ) + return cred_id + + +def _source_type_value(source_type: str) -> str: + """Onboarding source type -> DocumentSource value (for prioritized_sources).""" + return { + "web": DocumentSource.WEB.value, + "confluence": DocumentSource.CONFLUENCE.value, + "github": DocumentSource.GITHUB.value, + "slack": DocumentSource.SLACK.value, + "jira": DocumentSource.JIRA.value, + }.get(source_type, source_type) + + +def _build_prompt( + payload: dict, admin_user: User | None, db_session: Session +) -> Prompt: + """Create the team's prompt, defaulting to the Orchestrator persona's prompt + and overriding with whatever the requester edited in the form.""" + template = get_persona_by_name(_DEFAULT_TEMPLATE_PERSONA, admin_user, db_session) + default_system = "" + default_task = "" + if template and template.prompts: + default_system = template.prompts[0].system_prompt + default_task = template.prompts[0].task_prompt + team = payload["team_name"] + return upsert_prompt( + user=admin_user, + name=f"[onboarding] {team} prompt", + description=f"Prompt for the {team} assistant (from onboarding)", + system_prompt=payload.get("system_prompt") or default_system, + task_prompt=payload.get("task_prompt") or default_task, + include_citations=True, + datetime_aware=True, + personas=None, + db_session=db_session, + default_prompt=False, + ) + + +def _build_channel_config( + payload: dict, prioritized_sources: list[str] +) -> ChannelConfig: + channel_name = payload["channel"]["channel_name"] + config: ChannelConfig = { + "channel_names": [channel_name], + "respond_tag_only": bool(payload.get("respond_tag_only", False)), + "prioritized_sources": prioritized_sources, + } + sme = payload.get("sme") or {} + if sme.get("enabled"): + config["enable_sme_validation"] = True + config["sme_group_name"] = sme.get("group_name", "") + oncall = payload.get("oncall") or {} + if oncall.get("enabled"): + if oncall.get("schedule"): + config["opsgenie_schedule"] = oncall["schedule"] + # DRI Slack handles/emails to tag on "need more help" (follow_up_tags is + # resolved as emails->users then handles/names->user-groups at runtime). + handles = [ + h.strip().lstrip("@") + for h in (oncall.get("handles") or "").split(",") + if h.strip() + ] + if handles: + config["follow_up_tags"] = handles + jira = payload.get("jira") or {} + if jira.get("enabled"): + config["jira_config"] = { + "enable_jira_integration": True, + "project_key": jira.get("project_key", ""), + "issue_type": jira.get("issue_type", ""), + "component": jira.get("component", ""), + } + return config + + +def _prioritized_sources(sources: list[dict]) -> list[str]: + """Distinct DocumentSource values in the requester's priority order.""" + ordered: list[str] = [] + for s in sources: + st = _source_type_value(s["type"]) + if st not in ordered: + ordered.append(st) + return ordered + + +def _dedup_confluence_sources(sources: list[dict], db_session: Session) -> list[dict]: + """Drop child Confluence pages when a parent is also provided, keeping the + parent alone (the connector already recurses a page's descendants). A whole- + space URL supersedes any page in that space; a parent page supersedes its + descendant pages (best-effort via the Confluence ancestry API).""" + conf = [(i, s) for i, s in enumerate(sources) if s.get("type") == "confluence"] + if len(conf) < 2: + return sources + keys: dict[int, tuple[str, str]] = {} + for i, s in conf: + try: + _b, space, page_id, _c = extract_confluence_keys_from_url(s["value"]) + keys[i] = (space.lower(), page_id or "") + except Exception: + keys[i] = ("", "") + drop: set[int] = set() + + # A whole-space URL supersedes pages in the same space. + space_roots = {sp for (sp, pid) in keys.values() if sp and not pid} + for i, _s in conf: + sp, pid = keys[i] + if pid and sp in space_roots: + drop.add(i) + + # A parent page supersedes its descendant pages (best-effort). + remaining = [i for i, _s in conf if i not in drop and keys[i][1]] + if len(remaining) >= 2: + provided = {keys[i][1] for i in remaining} + ancestry = confluence_page_ancestors(list(provided), db_session) + for i in remaining: + if provided & set(ancestry.get(keys[i][1], [])): + drop.add(i) # an ancestor of this page was also provided + + if drop: + logger.info( + "onboarding: dropped %d redundant child Confluence source(s)", len(drop) + ) + return [s for i, s in enumerate(sources) if i not in drop] + + +def provision_onboarding( + request: OnboardingRequest, + admin_user: User, + db_session: Session, +) -> OnboardingRequest: + """Phase 1 (on approval): create the connectors + cc_pairs and kick off + high-priority indexing; status -> INDEXING. The document set, assistant, and + Slack config are created later by `finalize_onboarding`, once every source + has finished scraping (so Darwin only goes live once it has real knowledge). + On any failure here the request is marked FAILED and the exception re-raised.""" + payload = request.payload + update_onboarding_status( + db_session, request, OnboardingStatus.PROVISIONING, approver_id=admin_user.id + ) + try: + embedding_model = get_current_db_embedding_model(db_session) + team = payload["team_name"] + sources = _dedup_confluence_sources(list(payload["sources"]), db_session) + + cc_pair_ids: list[int] = [] + index_targets: list[tuple[int, int]] = [] # (connector_id, credential_id) + for source in sources: + # Preflight: the reusable credential must be able to reach the source. + _assert_source_accessible(source, db_session) + connector = create_connector( + _build_connector_base(source, db_session), db_session + ) + connector_id = int(connector.id) + credential_id = _credential_id_for_source(source["type"], db_session) + ccp = add_credential_to_connector( + connector_id=connector_id, + credential_id=credential_id, + cc_pair_name=f"[onboarding] {team}: {source.get('label') or source['value']}", + is_public=True, + user=admin_user, + db_session=db_session, + ) + if ccp.data is None: + raise ValueError(f"Failed to create cc_pair for {source['value']}") + # NB: in this fork add_credential_to_connector returns + # data=connector_id, NOT the cc_pair id. Look the pair up by + # (connector, credential) to record its real id — otherwise the + # request tracks connector ids as if they were cc_pair ids, which + # (off by one) points the finalizer/status at the wrong cc_pairs. + cc_pair = get_connector_credential_pair( + connector_id, credential_id, db_session + ) + if cc_pair is None: + raise ValueError(f"Failed to create cc_pair for {source['value']}") + cc_pair_ids.append(cc_pair.id) + index_targets.append((connector_id, credential_id)) + + set_provisioned_ids(db_session, request, cc_pair_ids=cc_pair_ids) + + # Kick off high-priority indexing for each source. + for connector_id, credential_id in index_targets: + create_index_attempt( + connector_id=connector_id, + credential_id=credential_id, + embedding_model_id=embedding_model.id, + db_session=db_session, + from_beginning=True, + indexing_priority=ONBOARDING_INDEXING_PRIORITY, + ) + + update_onboarding_status(db_session, request, OnboardingStatus.INDEXING) + logger.info( + "onboarding %s provisioned %d source(s); indexing. cc_pairs=%s", + request.id, + len(cc_pair_ids), + cc_pair_ids, + ) + # Customer thread: reviewed + approved, indexing started. + notify_client_approved(request) + return request + except Exception as e: + logger.exception("onboarding %s provisioning failed", request.id) + update_onboarding_status( + db_session, request, OnboardingStatus.FAILED, error_msg=str(e) + ) + raise + + +def _finalize_owner(request: OnboardingRequest, db_session: Session) -> User | None: + """Owner of the created assistant/doc-set — the approver, else the requester. + finalize runs in the background, so we resolve it from the row.""" + for uid in (request.approver_id, request.requester_id): + if uid is not None: + user = db_session.get(User, uid) + if user is not None: + return user + return None + + +def finalize_onboarding( + request: OnboardingRequest, db_session: Session +) -> OnboardingRequest: + """Phase 2 (after all sources are scraped): create the document set over the + request's cc_pairs, the assistant (prompt + persona) tied to it, and the Slack + bot config — making Darwin live in the channel — then mark COMPLETE + notify.""" + payload = request.payload + owner = _finalize_owner(request, db_session) + owner_id = owner.id if owner else None + team = payload["team_name"] + cc_pair_ids = list(request.cc_pair_ids or []) + # Capture already-provisioned artifact ids into locals BEFORE dropping the + # transaction: the idempotency guards below must not re-open one (a read + # would auto-begin) right before insert_document_set's begin(). + document_set_id = request.document_set_id + persona_id = request.persona_id + slack_bot_config_id = request.slack_bot_config_id + + # The finalizer already opened a transaction on this session doing its reads + # (list / _source_index_state / _finalize_owner above). insert_document_set + # calls db_session.begin(), which raises "A transaction is already begun" + # when one is active — so drop the read-only transaction here. + db_session.rollback() + + # Idempotent + incremental so this survives a pod crash mid-finalize: create + # each artifact only if the request doesn't already reference one, and + # persist its id immediately (a checkpoint). A crash between steps resumes on + # the next finalizer tick instead of duplicating doc sets / personas / Slack + # configs. Each helper commits, so every persisted id is durable. + + # 1. Document set over the request's cc_pairs. + if document_set_id is None: + doc_set, _ = insert_document_set( + DocumentSetCreationRequest( + name=f"[onboarding] {team}", + description=f"Sources onboarded for {team}", + cc_pair_ids=cc_pair_ids, + is_public=True, + ), + owner_id, + db_session, + ) + document_set_id = doc_set.id + set_provisioned_ids(db_session, request, document_set_id=document_set_id) + + # 2. Assistant (prompt + persona) tied to the document set. The SearchTool + # MUST be attached or the assistant has the document set but can't actually + # search it (upsert_persona only enables search when tool_ids includes it; + # the startup auto-add migration doesn't cover personas created later here). + if persona_id is None: + prompt = _build_prompt(payload, owner, db_session) + search_tool = ( + db_session.query(Tool) + .filter(Tool.in_code_tool_id == _SEARCH_TOOL_IN_CODE_ID) + .first() + ) + if search_tool is None: + logger.warning( + "onboarding %s: SearchTool not found — assistant will have the " + "document set but no search tool", + request.id, + ) + tool_ids = [search_tool.id] if search_tool else None + persona = upsert_persona( + user=owner, + name=team, + description=f"Assistant for {team} (self-serve onboarding)", + num_chunks=10, + llm_relevance_filter=False, + llm_filter_extraction=False, + recency_bias=RecencyBiasSetting.BASE_DECAY, + llm_model_provider_override=None, + llm_model_version_override=None, + starter_messages=None, + is_public=True, + db_session=db_session, + prompt_ids=[prompt.id], + document_set_ids=[document_set_id], + tool_ids=tool_ids, + ) + persona_id = persona.id + set_provisioned_ids(db_session, request, persona_id=persona_id) + + # 3. Slack bot config -> persona, making Darwin live in the channel. + if slack_bot_config_id is None: + channel_config = _build_channel_config( + payload, _prioritized_sources(payload["sources"]) + ) + response_type = ( + SlackBotResponseType.QUOTES + if payload.get("response_type") == "quotes" + else SlackBotResponseType.CITATIONS + ) + slack_config = insert_slack_bot_config( + persona_id=persona_id, + channel_config=channel_config, + response_type=response_type, + db_session=db_session, + ) + slack_bot_config_id = slack_config.id + set_provisioned_ids( + db_session, request, slack_bot_config_id=slack_bot_config_id + ) + + # 4. Mark live + notify. A notification failure must not fail finalize — the + # request is already COMPLETE (and would otherwise be retried forever). + update_onboarding_status(db_session, request, OnboardingStatus.COMPLETE) + logger.info( + "onboarding %s finalized: persona=%s doc_set=%s config=%s", + request.id, + persona_id, + document_set_id, + slack_bot_config_id, + ) + try: + notify_onboarding_complete(request) + notify_client_complete(request) + except Exception: + logger.exception( + "onboarding %s finalized but completion notification failed", request.id + ) + return request + + +def _source_index_state( + request: OnboardingRequest, db_session: Session +) -> tuple[bool, list[str], bool]: + """(all_indexed, failures, in_progress) from each cc_pair's LATEST index + attempt. all_indexed is True only when every source's latest run succeeded; + failures lists reasons for any source whose latest run failed; in_progress is + True when any source is still scraping (NOT_STARTED / IN_PROGRESS) — used to + move a previously-FAILED request back to INDEXING once it's re-indexing.""" + cc_pair_ids = request.cc_pair_ids or [] + if not cc_pair_ids: + return False, [], False + statuses: list[IndexingStatus | None] = [] + failures: list[str] = [] + for cc_id in cc_pair_ids: + cc = get_connector_credential_pair_from_id(cc_id, db_session) + if cc is None: + statuses.append(None) + continue + latest = db_session.execute( + select(IndexAttempt) + .where(IndexAttempt.connector_id == cc.connector_id) + .where(IndexAttempt.credential_id == cc.credential_id) + .order_by(desc(IndexAttempt.time_created)) + .limit(1) + ).scalar_one_or_none() + status = latest.status if latest else None + statuses.append(status) + if status == IndexingStatus.FAILED: + reason = (latest.error_msg if latest else None) or "indexing failed" + failures.append(f"{cc.name}: {reason[:150]}") + all_indexed = bool(statuses) and all(s == IndexingStatus.SUCCESS for s in statuses) + in_progress = any( + s in (IndexingStatus.NOT_STARTED, IndexingStatus.IN_PROGRESS) for s in statuses + ) + return all_indexed, failures, in_progress + + +def finalize_ready_onboarding_requests(db_session: Session) -> None: + """Background poll: for each request still being tracked (INDEXING or FAILED), + drive its status from the aggregate state of its sources. Ordering matters — + in-progress WINS over a failure so the requester never sees a scary FAILED + while work is still happening: + + * all sources succeeded -> finalize (assistant goes live) -> COMPLETE + * any source still scraping -> INDEXING (defer any failure verdict; + a stale FAILED is cleared back to + INDEXING so re-indexing shows progress) + * settled with >=1 failed source -> FAILED + notify ONCE + + Recoverable by design: fix + re-index a source and the request climbs back to + INDEXING and then COMPLETE on later ticks; nothing needs restarting. An + already-FAILED request that is still fully settled+failing is not re-notified.""" + tracked = list_onboarding_requests( + db_session, OnboardingStatus.INDEXING + ) + list_onboarding_requests(db_session, OnboardingStatus.FAILED) + for request in tracked: + try: + all_indexed, failures, in_progress = _source_index_state( + request, db_session + ) + if all_indexed: + finalize_onboarding(request, db_session) + elif in_progress: + # Something is still scraping (initial run or a re-index). Show + # INDEXING and hold off on any failure verdict — a transient + # failure may still be superseded by an in-flight or retried + # source. Only flips the row if it isn't already INDEXING (also + # clears a stale error_msg from a prior FAILED). + if request.status != OnboardingStatus.INDEXING.value: + update_onboarding_status( + db_session, request, OnboardingStatus.INDEXING + ) + elif failures and request.status != OnboardingStatus.FAILED.value: + # Everything settled and at least one source failed: flag FAILED + + # notify ONCE (an already-FAILED request is not re-notified). + detail = "; ".join(failures) + update_onboarding_status( + db_session, request, OnboardingStatus.FAILED, error_msg=detail + ) + notify_onboarding_failed(request, detail) + notify_client_failed(request) + except Exception: + logger.exception("onboarding %s finalize check failed", request.id) + db_session.rollback() diff --git a/backend/danswer/onboarding/validation.py b/backend/danswer/onboarding/validation.py new file mode 100644 index 00000000000..e53587bcbc2 --- /dev/null +++ b/backend/danswer/onboarding/validation.py @@ -0,0 +1,481 @@ +"""Inline validation for the self-serve onboarding form. + +Every source/handle a requester enters is validated against the live system +before submission: Slack channels + user-groups via the bot's Slack client, +Confluence spaces via an existing Confluence connector's credentials, and +docs.uipath.com roots via the web-connector's version logic. Each validator is +resilient — it distinguishes "invalid" from "couldn't check right now".""" +import re +from urllib.parse import urlparse + +from pydantic import BaseModel +from slack_sdk import WebClient +from slack_sdk.errors import SlackApiError +from sqlalchemy import select +from sqlalchemy.orm import Session + +from danswer.configs.app_configs import GITHUB_CONNECTOR_BASE_URL +from danswer.configs.constants import DocumentSource +from danswer.connectors.confluence.connector import extract_confluence_keys_from_url +from danswer.connectors.web.connector import _uipath_product_prefix +from danswer.danswerbot.slack.tokens import fetch_tokens +from danswer.danswerbot.slack.utils import fetch_groupids_from_names +from danswer.db.models import Connector +from danswer.db.models import Credential +from danswer.utils.logger import setup_logger + +logger = setup_logger() + +# Slack has no name->channel lookup API. On a large Enterprise Grid org a bare +# name can be thousands of channels deep, so instead of enumerating everything we +# search only the channels the bot is a MEMBER of (users.conversations — a small +# set), which is also the real precondition for the bot to operate in a channel. +_MEMBER_CHANNEL_PAGES = 6 # ~6k channels the bot is in — plenty +_MENTION_RE = re.compile(r"<#(C[A-Z0-9]+)(?:\|[^>]*)?>") +_CHANNEL_ID_RE = re.compile(r"^C[A-Z0-9]{6,}$") +# A pasted channel link, e.g. https://.slack.com/archives/C0123ABC/... +_ARCHIVES_RE = re.compile(r"/archives/(C[A-Z0-9]+)") + + +class ValidationResult(BaseModel): + valid: bool + message: str + # Resolved canonical values (e.g. channel id/name, wiki base) when valid. + resolved: dict = {} + + +def _bot_client() -> WebClient: + return WebClient(token=fetch_tokens().bot_token) + + +def _find_channel_by_name( + method: object, name: str, max_pages: int, **kwargs: object +) -> dict | None: + """Paginate a Slack list endpoint (conversations_list / users_conversations) + looking for an exact channel-name match.""" + cursor: str | None = None + for _ in range(max_pages): + resp = method(limit=1000, cursor=cursor, **kwargs) # type: ignore[operator] + for ch in resp.get("channels", []): + if ch.get("name", "").lower() == name: + return ch + meta = resp.get("response_metadata") + cursor = meta.get("next_cursor") if isinstance(meta, dict) else None + if not cursor: + break + return None + + +def validate_slack_channel(value: str) -> ValidationResult: + """Accepts a #mention, a channel id, or a bare name. Resolves to the real + channel and confirms the bot can see it.""" + raw = (value or "").strip() + if not raw: + return ValidationResult(valid=False, message="Enter a channel") + + mention = _MENTION_RE.search(raw) + link = _ARCHIVES_RE.search(raw) + channel_id = ( + mention.group(1) + if mention + else link.group(1) + if link + else raw + if _CHANNEL_ID_RE.match(raw) + else None + ) + try: + client = _bot_client() + except Exception as e: + logger.warning("slack client unavailable for validation: %s", e) + return ValidationResult(valid=False, message="Couldn't reach Slack to verify") + + if channel_id: + try: + ch = client.conversations_info(channel=channel_id)["channel"] + # Scraping needs the bot IN the channel. It auto-joins PUBLIC channels + # on scrape, but can't self-join PRIVATE ones — those need an invite. + if ch.get("is_member"): + note = "the bot is already in this channel" + elif ch.get("is_private"): + note = "invite @Darwin — private channels can't be auto-joined" + else: + note = "the bot will auto-join this public channel when it scrapes" + return ValidationResult( + valid=True, + message=f"#{ch['name']} · {note}", + resolved={"channel_id": ch["id"], "channel_name": ch["name"]}, + ) + except SlackApiError as e: + return ValidationResult( + valid=False, + message=( + "Channel not found or the bot can't access it — for a private " + f"channel, invite @Darwin first ({e.response.get('error')})" + ), + ) + + # Bare name. Slack has no name->channel lookup API, and every channel-read + # endpoint (conversations_info/history/replies) needs the ID — so there's no + # cheap way to confirm a channel the bot isn't in yet, and enumerating the + # whole org (tens of thousands of channels on Enterprise Grid) is too costly. + # Crucially, the bot is only added to the channel DURING onboarding, so a + # brand-new channel legitimately isn't a member yet. We therefore never block + # on existence: we do a cheap membership check purely as a positive signal + # (and to resolve the real id/name when we can), and otherwise accept the + # name — provisioning uses the name, and the connector auto-joins on index. + name = raw.lstrip("#").lower() + try: + ch = _find_channel_by_name( + client.users_conversations, + name, + _MEMBER_CHANNEL_PAGES, + types="public_channel,private_channel", + ) + except SlackApiError as e: + logger.warning( + "slack membership check failed for %s: %s", name, e.response.get("error") + ) + ch = None + if ch: + return ValidationResult( + valid=True, + message=f"#{ch['name']} · the bot is already in this channel", + resolved={"channel_id": ch["id"], "channel_name": ch["name"]}, + ) + return ValidationResult( + valid=True, + message=( + f"#{name} · public channels auto-join when scraped; " + "invite @Darwin first if it's private" + ), + resolved={"channel_name": name}, + ) + + +def validate_slack_group(name: str) -> ValidationResult: + """One or more Slack user-group handles/names (comma-separated) — used for SME + groups and the on-call DRI handles (e.g. "@as-dri"). Matches on either the + group name or its @handle; reports any that don't resolve.""" + handles = [h.strip().lstrip("@") for h in (name or "").split(",") if h.strip()] + if not handles: + return ValidationResult(valid=False, message="Enter a group handle") + try: + client = _bot_client() + group_ids, failed = fetch_groupids_from_names(handles, client) + except Exception as e: + logger.warning("slack group validation failed: %s", e) + return ValidationResult(valid=False, message="Couldn't reach Slack to verify") + if failed: + missing = ", ".join(f"@{h}" for h in failed) + return ValidationResult(valid=False, message=f"No Slack user group: {missing}") + return ValidationResult( + valid=True, + message=", ".join(f"@{h}" for h in handles), + resolved={"group_ids": group_ids}, + ) + + +def _first_confluence_credential(db_session: Session) -> dict | None: + """Reuse an existing Confluence connector's creds for inline space checks.""" + for cred in db_session.execute(select(Credential)).scalars(): + cj = cred.credential_json or {} + if cj.get("confluence_access_token") and cj.get("confluence_username"): + return cj + return None + + +def _allowed_confluence_hosts(db_session: Session) -> set[str]: + """Hosts of existing Confluence connectors — the ONLY hosts we'll send our + stored Confluence token to (SSRF / credential-leak guard: never let a + user-supplied URL point the authenticated client at an arbitrary host).""" + hosts: set[str] = set() + for connector in ( + db_session.execute( + select(Connector).where(Connector.source == DocumentSource.CONFLUENCE) + ) + .scalars() + .all() + ): + wiki_url = (connector.connector_specific_config or {}).get("wiki_page_url") + netloc = urlparse(wiki_url).netloc.lower() if wiki_url else "" + if netloc: + hosts.add(netloc) + return hosts + + +def _confluence_base_url(db_session: Session) -> str | None: + """Wiki base URL of an existing Confluence connector (to build a client).""" + for connector in ( + db_session.execute( + select(Connector).where(Connector.source == DocumentSource.CONFLUENCE) + ) + .scalars() + .all() + ): + wiki_url = (connector.connector_specific_config or {}).get("wiki_page_url") + if wiki_url: + try: + base, _s, _p, _c = extract_confluence_keys_from_url(wiki_url) + return base + except Exception: + continue + return None + + +def confluence_page_ancestors( + page_ids: list[str], db_session: Session +) -> dict[str, list[str]]: + """Map each Confluence page id -> list of its ancestor page ids (best-effort). + + Used to drop child pages when a parent is also provided (the connector + already recurses a page's descendants). Returns {} if we can't build a + client; individual failures resolve to an empty ancestor list.""" + base = _confluence_base_url(db_session) + creds = _first_confluence_credential(db_session) + if not base or creds is None: + return {} + result: dict[str, list[str]] = {} + try: + from atlassian import Confluence # type: ignore[import-untyped] + + client = Confluence( + url=base, + username=creds["confluence_username"], + password=creds["confluence_access_token"], + cloud="atlassian.net" in base, + ) + for pid in page_ids: + try: + ancestors = client.get_page_ancestors(pid) or [] + result[pid] = [str(a["id"]) for a in ancestors if a.get("id")] + except Exception as e: + logger.info("confluence ancestors lookup failed for %s: %s", pid, e) + result[pid] = [] + except Exception as e: + logger.info("confluence ancestry client unavailable: %s", e) + return {} + return result + + +def validate_confluence_url(url: str, db_session: Session) -> ValidationResult: + """Parse the wiki URL and confirm the target is reachable, using an existing + Confluence connector's credentials — but ONLY if the URL's host matches an + existing Confluence connector (never send creds to an arbitrary host). + + Scope depends on the URL, mirroring the connector: a PAGE url scrapes that + page + all its child pages; a SPACE url scrapes the whole space.""" + raw = (url or "").strip() + if not raw: + return ValidationResult(valid=False, message="Enter a Confluence URL") + try: + wiki_base, space, page_id, is_cloud = extract_confluence_keys_from_url(raw) + except ValueError: + return ValidationResult(valid=False, message="Not a valid Confluence wiki URL") + + # SSRF / credential-leak guard: only proceed against a known Confluence host. + host = urlparse(wiki_base).netloc.lower() + allowed_hosts = _allowed_confluence_hosts(db_session) + if allowed_hosts and host not in allowed_hosts: + return ValidationResult( + valid=False, + message=f"Confluence host not allowed — must be one of {sorted(allowed_hosts)}", + ) + + scope = ( + "this page and all its child pages" if page_id else f"the whole '{space}' space" + ) + resolved = { + "wiki_base": wiki_base, + "space": space, + "page_id": page_id, + "is_cloud": is_cloud, + } + + creds = _first_confluence_credential(db_session) + # No creds, or no known Confluence host to vet against -> parse-only (never + # send the token to an unvetted host). + if creds is None or not allowed_hosts: + return ValidationResult( + valid=True, + message=f"Will scrape {scope} (access unverified)", + resolved=resolved, + ) + try: + from atlassian import Confluence # type: ignore[import-untyped] + + client = Confluence( + url=wiki_base, + username=creds["confluence_username"], + password=creds["confluence_access_token"], + cloud=is_cloud, + ) + # Verify the actual target: the page for a page url, else the space. + if page_id: + client.get_page_by_id(page_id) + else: + client.get_space(space) + except Exception as e: + logger.info("confluence validation failed for %s: %s", raw, e) + target = "Page" if page_id else f"Space '{space}'" + return ValidationResult( + valid=False, message=f"{target} not found or inaccessible" + ) + return ValidationResult( + valid=True, message=f"Will scrape {scope}", resolved=resolved + ) + + +_GITHUB_HOSTS = {"github.com", "www.github.com"} + + +def _first_github_token(db_session: Session) -> str | None: + """Reuse an existing GitHub connector's token for the access check (the + requester never supplies one).""" + for cred in db_session.execute(select(Credential)).scalars(): + tok = (cred.credential_json or {}).get("github_access_token") + if tok: + return tok + return None + + +def validate_github_repo(url: str, db_session: Session) -> ValidationResult: + """Confirm a github.com repo URL is reachable with the stored GitHub token — + i.e. the credential that provisioning will reuse can actually scrape it. + + The token is only ever sent to the GitHub API (github.com or the configured + enterprise base URL), never to the URL's host. Errors are reported generically + and logged by exception type only, so a token or raw API payload can't leak + into a message or the logs.""" + raw = (url or "").strip() + if not raw: + return ValidationResult(valid=False, message="Enter a GitHub repo URL") + parsed = urlparse(raw) + host = (parsed.hostname or "").lower() + if host not in _GITHUB_HOSTS: + return ValidationResult(valid=False, message="Must be a github.com repo URL") + parts = [p for p in parsed.path.strip("/").split("/") if p] + if len(parts) < 2: + return ValidationResult( + valid=False, message="URL must be github.com//" + ) + owner, repo = parts[0], parts[1].removesuffix(".git") + full_name = f"{owner}/{repo}" + + token = _first_github_token(db_session) + if token is None: + return ValidationResult( + valid=True, + message=f"{full_name} (access unverified — no GitHub credential on file)", + resolved={"repo_owner": owner, "repo_name": repo}, + ) + try: + from github import Github # type: ignore[import-untyped] + + client = ( + Github(token, base_url=GITHUB_CONNECTOR_BASE_URL) + if GITHUB_CONNECTOR_BASE_URL + else Github(token) + ) + client.get_repo(full_name) # raises if the token can't see the repo + except Exception as e: + # Log the exception TYPE only — never str(e), which can echo the token + # or API payload. + logger.info( + "github repo access check failed for %s: %s", full_name, type(e).__name__ + ) + return ValidationResult( + valid=False, + message=f"Repo '{full_name}' not found or the GitHub app lacks access", + ) + return ValidationResult( + valid=True, + message=full_name, + resolved={"repo_owner": owner, "repo_name": repo}, + ) + + +def _first_jira_credential(db_session: Session) -> dict | None: + """Reuse an existing Jira connector's credential for the filter check.""" + for cred in db_session.execute(select(Credential)).scalars(): + cj = cred.credential_json or {} + if cj.get("jira_api_token"): + return cj + return None + + +def jira_base_url(db_session: Session) -> str | None: + """Base URL of an existing Jira connector (onboarding reuses it — the + requester supplies only the filter, never the host/creds).""" + for connector in ( + db_session.execute( + select(Connector).where(Connector.source == DocumentSource.JIRA) + ) + .scalars() + .all() + ): + base = (connector.connector_specific_config or {}).get("jira_base_url") + if base: + return base + return None + + +def validate_jira_filter(jql: str, db_session: Session) -> ValidationResult: + """Confirm a Jira JQL filter is valid and reachable with the stored Jira + credential (the same one provisioning will reuse). + + The credential is only ever sent to the existing Jira connector's base URL, + never a user-supplied host. Errors are reported generically and logged by + exception type only, so a token or JQL-echoing API payload can't leak.""" + raw = (jql or "").strip() + if not raw: + return ValidationResult(valid=False, message="Enter a Jira filter (JQL)") + + base = jira_base_url(db_session) + creds = _first_jira_credential(db_session) + if base is None or creds is None: + return ValidationResult( + valid=True, + message="Filter unverified — no Jira connector/credential on file", + resolved={"jira_filter": raw}, + ) + try: + from jira import JIRA # type: ignore[import-untyped] + + token = creds["jira_api_token"] + if creds.get("jira_user_email"): + client = JIRA(basic_auth=(creds["jira_user_email"], token), server=base) + else: + client = JIRA(token_auth=token, server=base) + # maxResults=1 keeps this cheap; an invalid JQL or access error raises. + client.search_issues(raw, maxResults=1) + except Exception as e: + logger.info("jira filter validation failed: %s", type(e).__name__) + return ValidationResult( + valid=False, message="Invalid Jira filter (JQL) or no access" + ) + return ValidationResult( + valid=True, message="Jira filter OK", resolved={"jira_filter": raw} + ) + + +def validate_docs_url(url: str) -> ValidationResult: + """A docs.uipath.com root URL. We normalize to the product root (version / + 'latest' segment stripped) since the web connector auto-crawls all versions.""" + raw = (url or "").strip() + if not raw: + return ValidationResult(valid=False, message="Enter a docs URL") + if not re.match(r"^https?://docs\.uipath\.com/", raw): + return ValidationResult(valid=False, message="Must be a docs.uipath.com URL") + from urllib.parse import urlparse + + parsed = urlparse(raw) + product_prefix = _uipath_product_prefix(parsed.path) + if not product_prefix.strip("/"): + return ValidationResult(valid=False, message="URL must include a product path") + root = f"{parsed.scheme}://{parsed.netloc}{product_prefix}" + return ValidationResult( + valid=True, + message=f"Will crawl all versions under {root}", + resolved={"root_url": root}, + ) diff --git a/backend/danswer/server/features/onboarding/__init__.py b/backend/danswer/server/features/onboarding/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/backend/danswer/server/features/onboarding/api.py b/backend/danswer/server/features/onboarding/api.py new file mode 100644 index 00000000000..aa12ca80302 --- /dev/null +++ b/backend/danswer/server/features/onboarding/api.py @@ -0,0 +1,358 @@ +"""Self-serve onboarding API. + +- basic_router (/onboarding): any authenticated user may submit a request, watch + their own requests, validate form fields inline, and monitor scrape status. +- admin_router (/admin/onboarding): admins list/approve/reject; approval triggers + provisioning (connectors + assistant + Slack config + high-priority indexing). +""" +from fastapi import APIRouter +from fastapi import Depends +from fastapi import HTTPException +from sqlalchemy import desc +from sqlalchemy import select +from sqlalchemy.orm import Session + +from danswer.auth.users import current_admin_user +from danswer.auth.users import current_user +from danswer.configs.app_configs import DISABLE_AUTH +from danswer.db.connector_credential_pair import get_connector_credential_pair_from_id +from danswer.db.engine import get_session +from danswer.db.models import IndexAttempt +from danswer.db.models import OnboardingRequest +from danswer.db.models import OnboardingStatus +from danswer.db.models import User +from danswer.db.models import UserRole +from danswer.db.onboarding import create_onboarding_request +from danswer.db.onboarding import get_onboarding_request +from danswer.db.onboarding import list_onboarding_requests +from danswer.db.onboarding import list_onboarding_requests_for_user +from danswer.db.onboarding import set_help_thread_ts +from danswer.db.onboarding import update_onboarding_payload +from danswer.db.onboarding import update_onboarding_status +from danswer.onboarding.notify import notify_client_submitted +from danswer.onboarding.notify import notify_onboarding_submitted +from danswer.onboarding.provision import provision_onboarding +from danswer.onboarding.validation import validate_confluence_url +from danswer.onboarding.validation import validate_docs_url +from danswer.onboarding.validation import validate_github_repo +from danswer.onboarding.validation import validate_jira_filter +from danswer.onboarding.validation import validate_slack_channel +from danswer.onboarding.validation import validate_slack_group +from danswer.onboarding.validation import ValidationResult +from danswer.server.features.onboarding.models import DecisionRequest +from danswer.server.features.onboarding.models import OnboardingRequestSnapshot +from danswer.server.features.onboarding.models import OnboardingStatusResponse +from danswer.server.features.onboarding.models import OnboardingSubmitRequest +from danswer.server.features.onboarding.models import SourceStatus +from danswer.server.features.onboarding.models import ValidateRequest + +basic_router = APIRouter(prefix="/onboarding") +admin_router = APIRouter(prefix="/admin/onboarding") + + +def _require_user(user: User | None) -> None: + """Onboarding is a human workflow — reject anonymous / API-key (service) + callers. `current_user` returns None for API-key callers and when auth is + globally disabled; allow None only in the latter (dev) case.""" + if user is None and not DISABLE_AUTH: + raise HTTPException(status_code=401, detail="Authentication required.") + + +# --- submit + validate + monitor (any authed user) -------------------------- + + +@basic_router.post("") +def submit_onboarding( + request: OnboardingSubmitRequest, + user: User | None = Depends(current_user), + db_session: Session = Depends(get_session), +) -> OnboardingRequestSnapshot: + _require_user(user) + if not request.sources: + raise HTTPException(status_code=400, detail="At least one source is required.") + created = create_onboarding_request( + db_session=db_session, + requester_id=user.id if user else None, + requester_email=user.email if user else "system@darwin", + payload=request.to_payload(), + ) + # Best-effort heads-up to the ops channel; never fail the submission on it. + notify_onboarding_submitted(created) + # Customer-facing thread root: @mention the requester + tracking link. Persist + # the returned thread ts so lifecycle updates reply in the same thread. + thread_ts = notify_client_submitted(created) + if thread_ts: + set_help_thread_ts(db_session, created, thread_ts) + return OnboardingRequestSnapshot.from_model(created) + + +@basic_router.post("/validate") +def validate_field( + request: ValidateRequest, + user: User | None = Depends(current_user), + db_session: Session = Depends(get_session), +) -> ValidationResult: + """Inline validation for a single form field.""" + _require_user(user) + if request.kind == "slack_channel": + return validate_slack_channel(request.value) + if request.kind == "slack_group": + return validate_slack_group(request.value) + if request.kind == "confluence": + return validate_confluence_url(request.value, db_session) + if request.kind == "github": + return validate_github_repo(request.value, db_session) + if request.kind == "jira": + return validate_jira_filter(request.value, db_session) + return validate_docs_url(request.value) + + +@basic_router.get("/default-prompt") +def default_prompt( + user: User | None = Depends(current_user), + db_session: Session = Depends(get_session), +) -> dict: + """The template prompt the form prefills (the Orchestrator assistant's prompt); + the requester can edit it before submitting.""" + from danswer.db.persona import get_persona_by_name + + template = get_persona_by_name("Orchestrator", user, db_session) + if template and template.prompts: + p = template.prompts[0] + return {"system_prompt": p.system_prompt, "task_prompt": p.task_prompt} + return {"system_prompt": "", "task_prompt": ""} + + +@basic_router.get("/mine") +def my_onboarding_requests( + user: User | None = Depends(current_user), + db_session: Session = Depends(get_session), +) -> list[OnboardingRequestSnapshot]: + if user is None: + return [] + return [ + OnboardingRequestSnapshot.from_model(r) + for r in list_onboarding_requests_for_user(db_session, user.id) + ] + + +def _source_statuses( + request: OnboardingRequest, db_session: Session +) -> list[SourceStatus]: + """Per-cc_pair scrape status for the request (from the latest IndexAttempt).""" + statuses: list[SourceStatus] = [] + for cc_pair_id in request.cc_pair_ids or []: + cc_pair = get_connector_credential_pair_from_id(cc_pair_id, db_session) + if cc_pair is None: + continue + latest = db_session.execute( + select(IndexAttempt) + .where(IndexAttempt.connector_id == cc_pair.connector_id) + .where(IndexAttempt.credential_id == cc_pair.credential_id) + .order_by(desc(IndexAttempt.time_created)) + .limit(1) + ).scalar_one_or_none() + statuses.append( + SourceStatus( + cc_pair_id=cc_pair_id, + name=cc_pair.name, + status=latest.status.value if latest else "not_started", + docs_indexed=(latest.total_docs_indexed or 0) if latest else 0, + error_msg=latest.error_msg if latest else None, + ) + ) + return statuses + + +@basic_router.get("/{request_id}/status") +def onboarding_status( + request_id: int, + user: User | None = Depends(current_user), + db_session: Session = Depends(get_session), +) -> OnboardingStatusResponse: + _require_user(user) + request = get_onboarding_request(db_session, request_id) + if request is None: + raise HTTPException(status_code=404, detail="Onboarding request not found.") + # Requester or an admin may view. + if ( + user is not None + and user.role != UserRole.ADMIN + and request.requester_id != user.id + ): + raise HTTPException(status_code=403, detail="Not your onboarding request.") + + # Status transitions (INDEXING -> COMPLETE / FAILED) are driven by the + # background finalizer, which also creates the doc set + assistant + Slack + # config once scraping finishes. This endpoint just reports current state. + sources = _source_statuses(request, db_session) + return OnboardingStatusResponse( + request_id=request.id, status=request.status, sources=sources + ) + + +def _require_owner_or_admin(request: OnboardingRequest, user: User | None) -> None: + _require_user(user) + if ( + user is not None + and user.role != UserRole.ADMIN + and request.requester_id != user.id + ): + raise HTTPException(status_code=403, detail="Not your onboarding request.") + + +@basic_router.get("/{request_id}") +def get_my_request( + request_id: int, + user: User | None = Depends(current_user), + db_session: Session = Depends(get_session), +) -> OnboardingRequestSnapshot: + """Full request (incl. payload) for the requester to open it in the form.""" + request = get_onboarding_request(db_session, request_id) + if request is None: + raise HTTPException(status_code=404, detail="Onboarding request not found.") + _require_owner_or_admin(request, user) + return OnboardingRequestSnapshot.from_model(request) + + +@basic_router.patch("/{request_id}") +def edit_my_request( + request_id: int, + body: OnboardingSubmitRequest, + user: User | None = Depends(current_user), + db_session: Session = Depends(get_session), +) -> OnboardingRequestSnapshot: + """Requester edits their own request while it's still pending.""" + request = get_onboarding_request(db_session, request_id) + if request is None: + raise HTTPException(status_code=404, detail="Onboarding request not found.") + _require_owner_or_admin(request, user) + if request.status != OnboardingStatus.PENDING.value: + raise HTTPException( + status_code=400, + detail=f"Request is '{request.status}', only pending requests can be edited.", + ) + if not body.sources: + raise HTTPException(status_code=400, detail="At least one source is required.") + update_onboarding_payload(db_session, request, body.to_payload()) + return OnboardingRequestSnapshot.from_model(request) + + +@basic_router.post("/{request_id}/cancel") +def cancel_my_request( + request_id: int, + user: User | None = Depends(current_user), + db_session: Session = Depends(get_session), +) -> OnboardingRequestSnapshot: + """Requester withdraws their own request while it's still pending.""" + request = get_onboarding_request(db_session, request_id) + if request is None: + raise HTTPException(status_code=404, detail="Onboarding request not found.") + _require_owner_or_admin(request, user) + if request.status != OnboardingStatus.PENDING.value: + raise HTTPException( + status_code=400, + detail=f"Request is '{request.status}', only pending requests can be cancelled.", + ) + update_onboarding_status(db_session, request, OnboardingStatus.CANCELLED) + return OnboardingRequestSnapshot.from_model(request) + + +# --- admin approval --------------------------------------------------------- + + +@admin_router.get("") +def list_requests( + _: User | None = Depends(current_admin_user), + db_session: Session = Depends(get_session), +) -> list[OnboardingRequestSnapshot]: + return [ + OnboardingRequestSnapshot.from_model(r) + for r in list_onboarding_requests(db_session) + ] + + +@admin_router.get("/{request_id}") +def get_request( + request_id: int, + _: User | None = Depends(current_admin_user), + db_session: Session = Depends(get_session), +) -> OnboardingRequestSnapshot: + """Full request (incl. payload) so an admin can open it in the form to edit.""" + request = get_onboarding_request(db_session, request_id) + if request is None: + raise HTTPException(status_code=404, detail="Onboarding request not found.") + return OnboardingRequestSnapshot.from_model(request) + + +@admin_router.patch("/{request_id}") +def edit_request( + request_id: int, + body: OnboardingSubmitRequest, + _: User | None = Depends(current_admin_user), + db_session: Session = Depends(get_session), +) -> OnboardingRequestSnapshot: + """Admin edits a PENDING request (fix gaps) before approving.""" + request = get_onboarding_request(db_session, request_id) + if request is None: + raise HTTPException(status_code=404, detail="Onboarding request not found.") + if request.status != OnboardingStatus.PENDING.value: + raise HTTPException( + status_code=400, + detail=f"Request is '{request.status}', only pending requests can be edited.", + ) + if not body.sources: + raise HTTPException(status_code=400, detail="At least one source is required.") + update_onboarding_payload(db_session, request, body.to_payload()) + return OnboardingRequestSnapshot.from_model(request) + + +@admin_router.post("/{request_id}/approve") +def approve_request( + request_id: int, + admin: User | None = Depends(current_admin_user), + db_session: Session = Depends(get_session), +) -> OnboardingRequestSnapshot: + request = get_onboarding_request(db_session, request_id) + if request is None: + raise HTTPException(status_code=404, detail="Onboarding request not found.") + if request.status != OnboardingStatus.PENDING.value: + raise HTTPException( + status_code=400, + detail=f"Request is '{request.status}', only pending requests can be approved.", + ) + if admin is None: + raise HTTPException( + status_code=403, detail="Admin identity required to approve." + ) + try: + provision_onboarding(request, admin, db_session) + except Exception as e: + # provision_onboarding already marked the request FAILED; surface the error. + raise HTTPException(status_code=500, detail=f"Provisioning failed: {e}") + return OnboardingRequestSnapshot.from_model(request) + + +@admin_router.post("/{request_id}/reject") +def reject_request( + request_id: int, + decision: DecisionRequest, + admin: User | None = Depends(current_admin_user), + db_session: Session = Depends(get_session), +) -> OnboardingRequestSnapshot: + request = get_onboarding_request(db_session, request_id) + if request is None: + raise HTTPException(status_code=404, detail="Onboarding request not found.") + if request.status != OnboardingStatus.PENDING.value: + raise HTTPException( + status_code=400, + detail=f"Request is '{request.status}', only pending requests can be rejected.", + ) + update_onboarding_status( + db_session, + request, + OnboardingStatus.REJECTED, + approver_id=admin.id if admin else None, + decision_reason=decision.reason, + ) + return OnboardingRequestSnapshot.from_model(request) diff --git a/backend/danswer/server/features/onboarding/models.py b/backend/danswer/server/features/onboarding/models.py new file mode 100644 index 00000000000..315163e722c --- /dev/null +++ b/backend/danswer/server/features/onboarding/models.py @@ -0,0 +1,112 @@ +"""Request/response models for the self-serve onboarding API.""" +import datetime +from typing import Literal + +from pydantic import BaseModel + +from danswer.db.models import OnboardingRequest + + +class ChannelRef(BaseModel): + channel_id: str + channel_name: str + + +class OnboardingSourceModel(BaseModel): + # Priority is the order in the sources list (index 0 = highest). + type: Literal["web", "confluence", "github", "slack", "jira"] + # docs root URL / confluence URL / github repo URL / slack channel / jira JQL + value: str + label: str | None = None + + +class SmeOption(BaseModel): + enabled: bool = False + group_name: str = "" # comma-separated Slack user-group names/@handles + + +class OncallOption(BaseModel): + enabled: bool = False + schedule: str = "" # opsgenie schedule name to tag on "need more help" + # comma-separated Slack user-group handles / emails to tag as DRI (e.g. @as-dri) + handles: str = "" + + +class JiraOption(BaseModel): + enabled: bool = False + project_key: str = "" + issue_type: str = "" + component: str = "" + + +class OnboardingSubmitRequest(BaseModel): + team_name: str + channel: ChannelRef + response_type: Literal["citations", "quotes"] = "citations" + respond_tag_only: bool = False + system_prompt: str = "" + task_prompt: str = "" + sme: SmeOption = SmeOption() + oncall: OncallOption = OncallOption() + jira: JiraOption = JiraOption() + sources: list[OnboardingSourceModel] + + def to_payload(self) -> dict: + return self.dict() + + +class OnboardingRequestSnapshot(BaseModel): + id: int + requester_email: str + status: str + payload: dict + decision_reason: str | None + error_msg: str | None + persona_id: int | None + document_set_id: int | None + slack_bot_config_id: int | None + cc_pair_ids: list[int] | None + created_at: datetime.datetime + updated_at: datetime.datetime + + @classmethod + def from_model(cls, r: OnboardingRequest) -> "OnboardingRequestSnapshot": + return cls( + id=r.id, + requester_email=r.requester_email, + status=r.status, + payload=r.payload, + decision_reason=r.decision_reason, + error_msg=r.error_msg, + persona_id=r.persona_id, + document_set_id=r.document_set_id, + slack_bot_config_id=r.slack_bot_config_id, + cc_pair_ids=r.cc_pair_ids, + created_at=r.created_at, + updated_at=r.updated_at, + ) + + +class ValidateRequest(BaseModel): + kind: Literal[ + "slack_channel", "slack_group", "confluence", "github", "jira", "docs" + ] + value: str + + +class DecisionRequest(BaseModel): + reason: str | None = None + + +class SourceStatus(BaseModel): + cc_pair_id: int + name: str + status: str # IndexingStatus value, or "not_started" + docs_indexed: int + error_msg: str | None + + +class OnboardingStatusResponse(BaseModel): + request_id: int + status: str + sources: list[SourceStatus] diff --git a/backend/tests/unit/danswer/connectors/slack/test_normalize_slack_link.py b/backend/tests/unit/danswer/connectors/slack/test_normalize_slack_link.py new file mode 100644 index 00000000000..d51ff844717 --- /dev/null +++ b/backend/tests/unit/danswer/connectors/slack/test_normalize_slack_link.py @@ -0,0 +1,110 @@ +import pytest +from slack_sdk.errors import SlackApiError + +from danswer.connectors.slack.utils import normalize_slack_link +from danswer.connectors.slack.utils import resolve_workspace_subdomain + + +class _FakeResp(dict): + """Minimal stand-in for slack_sdk's SlackResponse (dict-like + validate()).""" + + def validate(self) -> "_FakeResp": + return self + + +class _FakeClient: + def __init__(self, permalink: str | None = None, error: str | None = None): + self._permalink = permalink + self._error = error + + def chat_getPermalink(self, channel: str, message_ts: str) -> _FakeResp: + if self._error is not None: + raise SlackApiError(self._error, {"ok": False, "error": self._error}) + return _FakeResp({"ok": True, "permalink": self._permalink}) + + +@pytest.mark.parametrize( + "raw, expected", + [ + # The bug: workspace display name "Product" -> dead product.slack.com. + ( + "https://Product.slack.com/archives/C02EF2C5KS8/p1712595481616829", + "https://uipath-product.slack.com/archives/C02EF2C5KS8/p1712595481616829", + ), + # Trailing space variant seen in the data ("Product "). + ( + "https://Product .slack.com/archives/C02/p1?thread_ts=1712595481.616829", + "https://uipath-product.slack.com/archives/C02/p1?thread_ts=1712595481.616829", + ), + # Case-insensitive match. + ( + "https://product.slack.com/archives/C02/p1", + "https://uipath-product.slack.com/archives/C02/p1", + ), + # Idempotent: already-canonical link is untouched. + ( + "https://uipath-product.slack.com/archives/C02/p1", + "https://uipath-product.slack.com/archives/C02/p1", + ), + # Genuinely different workspaces must NOT be rewritten. + ( + "https://uipath-customer-ops.slack.com/archives/C99/p2", + "https://uipath-customer-ops.slack.com/archives/C99/p2", + ), + ( + "https://uipath-marketing.slack.com/archives/C99/p2", + "https://uipath-marketing.slack.com/archives/C99/p2", + ), + # Non-Slack links pass through unchanged (even if a path says "product"). + ( + "https://docs.uipath.com/product/latest", + "https://docs.uipath.com/product/latest", + ), + # Only the host is rewritten, not a matching path segment. + ( + "https://Product.slack.com/archives/product/p1", + "https://uipath-product.slack.com/archives/product/p1", + ), + ], +) +def test_normalize_slack_link(raw: str, expected: str) -> None: + assert normalize_slack_link(raw) == expected + + +def test_normalize_slack_link_empty() -> None: + assert normalize_slack_link("") == "" + + +def test_resolve_workspace_subdomain_from_permalink() -> None: + client = _FakeClient( + permalink="https://uipath-product.slack.com/archives/C02/p1712595481616829" + ) + assert ( + resolve_workspace_subdomain(client, "C02", "1712595481.616829", fallback="X") # type: ignore[arg-type] + == "uipath-product" + ) + + +def test_resolve_workspace_subdomain_other_workspace() -> None: + client = _FakeClient( + permalink="https://uipath-customer-ops.slack.com/archives/C99/p1" + ) + assert ( + resolve_workspace_subdomain(client, "C99", "1.1", fallback="X") # type: ignore[arg-type] + == "uipath-customer-ops" + ) + + +@pytest.mark.parametrize( + "client", + [ + _FakeClient(error="channel_not_found"), # API error -> fallback + _FakeClient(permalink=""), # empty permalink -> fallback + _FakeClient(permalink="https://example.com/not-slack"), # non-slack -> fallback + ], +) +def test_resolve_workspace_subdomain_falls_back(client: _FakeClient) -> None: + assert ( + resolve_workspace_subdomain(client, "C1", "1.1", fallback="uipath-product") # type: ignore[arg-type] + == "uipath-product" + ) diff --git a/backend/tests/unit/danswer/onboarding/__init__.py b/backend/tests/unit/danswer/onboarding/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/backend/tests/unit/danswer/onboarding/test_finalizer.py b/backend/tests/unit/danswer/onboarding/test_finalizer.py new file mode 100644 index 00000000000..5020c79571c --- /dev/null +++ b/backend/tests/unit/danswer/onboarding/test_finalizer.py @@ -0,0 +1,527 @@ +"""Unit tests for the background onboarding finalizer state machine. + +Covers `_source_index_state` (reading per-source scrape status, incl. the +in-progress flag) and `finalize_ready_onboarding_requests` (finalize on success, +flag failure once, recover a previously-FAILED request, and flip a re-indexing +FAILED request back to INDEXING) — positive and negative paths.""" +from types import SimpleNamespace +from typing import Any + +import pytest + +from danswer.db.enums import IndexingStatus +from danswer.db.models import OnboardingStatus +from danswer.onboarding import provision + + +# --- _source_index_state --------------------------------------------------- + + +class _Result: + def __init__(self, value: object) -> None: + self._value = value + + def scalar_one_or_none(self) -> object: + return self._value + + +class _Session: + """Returns queued latest-attempt rows, one per execute() call (one per cc).""" + + def __init__(self, attempts: list) -> None: + self._attempts = list(attempts) + + def execute(self, *args: object, **kwargs: object) -> _Result: + return _Result(self._attempts.pop(0) if self._attempts else None) + + +def _attempt(status: IndexingStatus, err: str | None = None) -> SimpleNamespace: + return SimpleNamespace(status=status, error_msg=err) + + +def _patch_cc(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + provision, + "get_connector_credential_pair_from_id", + lambda cc_id, db: SimpleNamespace( + connector_id=cc_id, credential_id=cc_id, name=f"src-{cc_id}" + ), + ) + + +def test_state_all_success(monkeypatch: pytest.MonkeyPatch) -> None: + _patch_cc(monkeypatch) + req = SimpleNamespace(cc_pair_ids=[10, 11]) + sess = _Session( + [_attempt(IndexingStatus.SUCCESS), _attempt(IndexingStatus.SUCCESS)] + ) + all_indexed, failures, in_progress = provision._source_index_state(req, sess) # type: ignore[arg-type] + assert all_indexed is True + assert failures == [] + assert in_progress is False + + +def test_state_one_failed(monkeypatch: pytest.MonkeyPatch) -> None: + _patch_cc(monkeypatch) + req = SimpleNamespace(cc_pair_ids=[10, 11]) + sess = _Session( + [_attempt(IndexingStatus.SUCCESS), _attempt(IndexingStatus.FAILED, "boom")] + ) + all_indexed, failures, in_progress = provision._source_index_state(req, sess) # type: ignore[arg-type] + assert all_indexed is False + assert len(failures) == 1 and "boom" in failures[0] + assert in_progress is False + + +def test_state_in_progress_flag_when_running(monkeypatch: pytest.MonkeyPatch) -> None: + _patch_cc(monkeypatch) + req = SimpleNamespace(cc_pair_ids=[10, 11]) + sess = _Session( + [_attempt(IndexingStatus.SUCCESS), _attempt(IndexingStatus.IN_PROGRESS)] + ) + all_indexed, failures, in_progress = provision._source_index_state(req, sess) # type: ignore[arg-type] + assert all_indexed is False # still scraping + assert failures == [] # not a failure either + assert in_progress is True + + +def test_state_in_progress_flag_when_not_started( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # A freshly-queued (NOT_STARTED) attempt also counts as in-progress: it's + # what a just-triggered re-index looks like before a worker picks it up. + _patch_cc(monkeypatch) + req = SimpleNamespace(cc_pair_ids=[10]) + sess = _Session([_attempt(IndexingStatus.NOT_STARTED)]) + all_indexed, failures, in_progress = provision._source_index_state(req, sess) # type: ignore[arg-type] + assert all_indexed is False + assert failures == [] + assert in_progress is True + + +def test_state_no_attempt_yet_is_not_in_progress( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # No attempt row at all (None) is neither done nor actively running. + _patch_cc(monkeypatch) + req = SimpleNamespace(cc_pair_ids=[10]) + sess = _Session([None]) # no index attempt row yet + all_indexed, failures, in_progress = provision._source_index_state(req, sess) # type: ignore[arg-type] + assert all_indexed is False + assert failures == [] + assert in_progress is False + + +def test_state_no_cc_pairs() -> None: + req = SimpleNamespace(cc_pair_ids=[]) + assert provision._source_index_state(req, None) == ( # type: ignore[arg-type] + False, + [], + False, + ) + + +# --- finalize_ready_onboarding_requests ------------------------------------ + + +def _wire( + monkeypatch: pytest.MonkeyPatch, + *, + indexing: list, + failed: list, + state: Any, # callable(request) -> (all_indexed, failures, in_progress) +) -> dict: + """Patch the finalizer's collaborators and capture what it does.""" + calls: dict = {"finalized": [], "failed_notified": [], "status": []} + + def _list(db: object, status: OnboardingStatus) -> list: + return indexing if status == OnboardingStatus.INDEXING else failed + + monkeypatch.setattr(provision, "list_onboarding_requests", _list) + monkeypatch.setattr(provision, "_source_index_state", lambda r, db: state(r)) + monkeypatch.setattr( + provision, "finalize_onboarding", lambda r, db: calls["finalized"].append(r.id) + ) + monkeypatch.setattr( + provision, + "notify_onboarding_failed", + lambda r, detail: calls["failed_notified"].append((r.id, detail)), + ) + + def _update( + db: object, r: object, status: OnboardingStatus, **kw: object + ) -> object: + r.status = status.value # type: ignore[attr-defined] + calls["status"].append((r.id, status.value)) # type: ignore[attr-defined] + return r + + monkeypatch.setattr(provision, "update_onboarding_status", _update) + return calls + + +def _req(id_: int, status: OnboardingStatus) -> SimpleNamespace: + return SimpleNamespace(id=id_, status=status.value, cc_pair_ids=[1]) + + +def test_finalizes_when_all_indexed(monkeypatch: pytest.MonkeyPatch) -> None: + req = _req(1, OnboardingStatus.INDEXING) + calls = _wire( + monkeypatch, indexing=[req], failed=[], state=lambda r: (True, [], False) + ) + provision.finalize_ready_onboarding_requests(None) # type: ignore[arg-type] + assert calls["finalized"] == [1] + assert calls["failed_notified"] == [] + + +def test_flags_failure_once(monkeypatch: pytest.MonkeyPatch) -> None: + req = _req(2, OnboardingStatus.INDEXING) + calls = _wire( + monkeypatch, + indexing=[req], + failed=[], + state=lambda r: (False, ["src-1: boom"], False), + ) + provision.finalize_ready_onboarding_requests(None) # type: ignore[arg-type] + assert (2, OnboardingStatus.FAILED.value) in calls["status"] + assert calls["failed_notified"] == [(2, "src-1: boom")] + assert calls["finalized"] == [] + + +def test_already_failed_not_renotified(monkeypatch: pytest.MonkeyPatch) -> None: + # A FAILED request that is still failing (nothing re-indexing) must not spam + # notifications and must stay FAILED. + req = _req(3, OnboardingStatus.FAILED) + calls = _wire( + monkeypatch, + indexing=[], + failed=[req], + state=lambda r: (False, ["still bad"], False), + ) + provision.finalize_ready_onboarding_requests(None) # type: ignore[arg-type] + assert calls["failed_notified"] == [] + assert calls["finalized"] == [] + assert calls["status"] == [] # no transition + + +def test_recovers_previously_failed(monkeypatch: pytest.MonkeyPatch) -> None: + # A FAILED request whose sources now all succeed is finalized — no restart. + req = _req(4, OnboardingStatus.FAILED) + calls = _wire( + monkeypatch, indexing=[], failed=[req], state=lambda r: (True, [], False) + ) + provision.finalize_ready_onboarding_requests(None) # type: ignore[arg-type] + assert calls["finalized"] == [4] + + +def test_failed_reindexing_flips_back_to_indexing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # A FAILED request that is now re-indexing (sources fixed + re-run) with + # nothing currently failing flips back to INDEXING so the requester sees + # progress — not finalized yet, not re-notified. + req = _req(8, OnboardingStatus.FAILED) + calls = _wire( + monkeypatch, indexing=[], failed=[req], state=lambda r: (False, [], True) + ) + provision.finalize_ready_onboarding_requests(None) # type: ignore[arg-type] + assert (8, OnboardingStatus.INDEXING.value) in calls["status"] + assert calls["finalized"] == [] + assert calls["failed_notified"] == [] + + +def test_failed_partial_reindex_flips_to_indexing_despite_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # In-progress WINS over a failure: a FAILED request with a source still + # scraping flips to INDEXING (work is happening) even though another source's + # latest attempt is failed — no re-notify. The failure verdict is deferred + # until everything settles. + req = _req(9, OnboardingStatus.FAILED) + calls = _wire( + monkeypatch, + indexing=[], + failed=[req], + state=lambda r: (False, ["src-2: still bad"], True), + ) + provision.finalize_ready_onboarding_requests(None) # type: ignore[arg-type] + assert (9, OnboardingStatus.INDEXING.value) in calls["status"] + assert calls["failed_notified"] == [] + assert calls["finalized"] == [] + + +def test_indexing_with_a_failure_but_still_running_stays_indexing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # A source failed while a sibling is still scraping: don't prematurely flag + # FAILED — stay INDEXING and wait for everything to settle. + req = _req(10, OnboardingStatus.INDEXING) + calls = _wire( + monkeypatch, + indexing=[req], + failed=[], + state=lambda r: (False, ["src-1: boom"], True), + ) + provision.finalize_ready_onboarding_requests(None) # type: ignore[arg-type] + assert calls["status"] == [] # already INDEXING, no flip needed + assert calls["failed_notified"] == [] # not flagged failed yet + assert calls["finalized"] == [] + + +def test_still_indexing_does_nothing(monkeypatch: pytest.MonkeyPatch) -> None: + req = _req(5, OnboardingStatus.INDEXING) + calls = _wire( + monkeypatch, indexing=[req], failed=[], state=lambda r: (False, [], True) + ) + provision.finalize_ready_onboarding_requests(None) # type: ignore[arg-type] + assert calls["finalized"] == [] + assert calls["failed_notified"] == [] + assert calls["status"] == [] + + +# --- finalize_onboarding: idempotent, crash-safe artifact creation ---------- + + +class _ToolQuery: + """Stub for db_session.query(Tool).filter(...).first() -> the SearchTool.""" + + def filter(self, *a: object, **k: object) -> "_ToolQuery": + return self + + def first(self) -> object: + return SimpleNamespace(id=99, in_code_tool_id="SearchTool") + + +class _FakeSession: + """Records rollback so we can assert the read txn is dropped before begin().""" + + def __init__(self, order: list) -> None: + self._order = order + + def query(self, *a: object, **k: object) -> _ToolQuery: + return _ToolQuery() + + def rollback(self) -> None: + self._order.append("rollback") + + def commit(self) -> None: + pass + + +def _finalize_request(**over: object) -> SimpleNamespace: + base: dict = dict( + id=1, + payload={"team_name": "T", "sources": [], "response_type": "citations"}, + cc_pair_ids=[1, 2], + approver_id=1, + requester_id=2, + document_set_id=None, + persona_id=None, + slack_bot_config_id=None, + status=OnboardingStatus.INDEXING.value, + ) + base.update(over) + return SimpleNamespace(**base) + + +def _wire_finalize( + monkeypatch: pytest.MonkeyPatch, + *, + order: list, + fail_on: str | None = None, + notify_raises: bool = False, +) -> dict: + """Patch finalize_onboarding's collaborators. Records step order; can inject a + failure at a given step; set_provisioned_ids writes ids back onto the request + so a follow-up call sees the resumed (checkpointed) state.""" + calls: dict = {"notified": 0} + monkeypatch.setattr( + provision, "_finalize_owner", lambda r, db: SimpleNamespace(id=9) + ) + + def _set_ids(db: object, req: object, **kw: object) -> object: + for k, v in kw.items(): + setattr(req, k, v) + return req + + monkeypatch.setattr(provision, "set_provisioned_ids", _set_ids) + + def _doc(req: object, uid: object, db: object) -> tuple: + order.append("doc_set") + if fail_on == "doc_set": + raise RuntimeError("boom doc_set") + return SimpleNamespace(id=101), [] + + monkeypatch.setattr(provision, "insert_document_set", _doc) + monkeypatch.setattr( + provision, "_build_prompt", lambda p, o, db: SimpleNamespace(id=2) + ) + + def _persona(**kw: object) -> object: + order.append("persona") + calls["persona_tool_ids"] = kw.get("tool_ids") + if fail_on == "persona": + raise RuntimeError("boom persona") + return SimpleNamespace(id=202) + + monkeypatch.setattr(provision, "upsert_persona", _persona) + monkeypatch.setattr(provision, "_build_channel_config", lambda p, ps: {}) + monkeypatch.setattr(provision, "_prioritized_sources", lambda s: []) + + def _slack(**kw: object) -> object: + order.append("slack") + if fail_on == "slack": + raise RuntimeError("boom slack") + return SimpleNamespace(id=303) + + monkeypatch.setattr(provision, "insert_slack_bot_config", _slack) + + def _status(db: object, req: object, status: object, **kw: object) -> object: + req.status = status.value # type: ignore[attr-defined] + order.append(f"status:{status.value}") # type: ignore[attr-defined] + return req + + monkeypatch.setattr(provision, "update_onboarding_status", _status) + + def _notify(req: object) -> None: + calls["notified"] += 1 + if notify_raises: + raise RuntimeError("slack down") + + monkeypatch.setattr(provision, "notify_onboarding_complete", _notify) + return calls + + +def test_finalize_creates_all_artifacts_in_order( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Happy path: rollback first (so insert_document_set can begin() its own + # transaction), then doc set -> persona -> slack -> COMPLETE, notify once. + order: list[str] = [] + calls = _wire_finalize(monkeypatch, order=order) + req = _finalize_request() + provision.finalize_onboarding(req, _FakeSession(order)) # type: ignore[arg-type] + assert order == ["rollback", "doc_set", "persona", "slack", "status:complete"] + assert req.document_set_id == 101 + assert req.persona_id == 202 + assert req.slack_bot_config_id == 303 + assert req.status == OnboardingStatus.COMPLETE.value + assert calls["notified"] == 1 + # The assistant must get the SearchTool, else it has the doc set but can't + # actually search it. + assert calls["persona_tool_ids"] == [99] + + +def test_finalize_drops_read_txn_before_document_set( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Regression: insert_document_set calls db_session.begin(), which raises if a + # transaction is already active (the finalizer opened one doing its reads). + # The rollback must happen before the document set is created. + order: list[str] = [] + _wire_finalize(monkeypatch, order=order) + provision.finalize_onboarding(_finalize_request(), _FakeSession(order)) # type: ignore[arg-type] + assert order.index("rollback") < order.index("doc_set") + + +def test_finalize_skips_document_set_when_already_created( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Idempotent: a request that already has a document_set_id must not create a + # second one (the duplicate-artifact bug a pod crash would otherwise cause). + order: list[str] = [] + _wire_finalize(monkeypatch, order=order) + req = _finalize_request(document_set_id=999) + provision.finalize_onboarding(req, _FakeSession(order)) # type: ignore[arg-type] + assert "doc_set" not in order + assert order == ["rollback", "persona", "slack", "status:complete"] + assert req.document_set_id == 999 # unchanged + + +def test_finalize_resumes_only_remaining_step( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Resume: doc set + persona already done -> only the Slack config is created. + order: list[str] = [] + _wire_finalize(monkeypatch, order=order) + req = _finalize_request(document_set_id=999, persona_id=888) + provision.finalize_onboarding(req, _FakeSession(order)) # type: ignore[arg-type] + assert order == ["rollback", "slack", "status:complete"] + assert req.slack_bot_config_id == 303 + + +def test_finalize_recovers_after_crash_without_duplicating( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Crash-safety: the first attempt creates doc set + persona (ids persisted) + # then dies at the Slack step. The retry must NOT recreate doc set / persona + # — it resumes at Slack and completes. + order1: list[str] = [] + _wire_finalize(monkeypatch, order=order1, fail_on="slack") + req = _finalize_request() + with pytest.raises(RuntimeError): + provision.finalize_onboarding(req, _FakeSession(order1)) # type: ignore[arg-type] + # Checkpoints persisted before the crash: + assert req.document_set_id == 101 + assert req.persona_id == 202 + assert req.slack_bot_config_id is None + assert req.status != OnboardingStatus.COMPLETE.value + + # Next finalizer tick (no injected failure): resumes at Slack only. + order2: list[str] = [] + calls2 = _wire_finalize(monkeypatch, order=order2) + provision.finalize_onboarding(req, _FakeSession(order2)) # type: ignore[arg-type] + assert order2 == ["rollback", "slack", "status:complete"] + assert "doc_set" not in order2 and "persona" not in order2 + assert req.slack_bot_config_id == 303 + assert req.status == OnboardingStatus.COMPLETE.value + assert calls2["notified"] == 1 + + +def test_finalize_completes_even_if_notify_fails( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # A completion-notification failure must not fail finalize — the request is + # already COMPLETE and would otherwise be retried (and re-notified) forever. + order: list[str] = [] + calls = _wire_finalize(monkeypatch, order=order, notify_raises=True) + req = _finalize_request() + provision.finalize_onboarding(req, _FakeSession(order)) # type: ignore[arg-type] + assert req.status == OnboardingStatus.COMPLETE.value + assert calls["notified"] == 1 + + +def test_finalizer_isolates_finalize_exception( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # If finalize_onboarding throws, the sweep must swallow it (rollback + log) + # so the request stays tracked for the next tick and other requests proceed. + req = _req(12, OnboardingStatus.INDEXING) + calls = _wire( + monkeypatch, indexing=[req], failed=[], state=lambda r: (True, [], False) + ) + + def _boom(r: object, db: object) -> None: + raise RuntimeError("finalize kaboom") + + monkeypatch.setattr(provision, "finalize_onboarding", _boom) + db = SimpleNamespace(rollback=lambda: calls.setdefault("rolled_back", True)) + # Must not raise. + provision.finalize_ready_onboarding_requests(db) # type: ignore[arg-type] + assert calls.get("rolled_back") is True + + +def test_per_request_error_is_isolated(monkeypatch: pytest.MonkeyPatch) -> None: + # One request blowing up must not stop the others from being processed. + good = _req(6, OnboardingStatus.INDEXING) + bad = _req(7, OnboardingStatus.INDEXING) + + def _state(r: object) -> tuple: + if r.id == 7: # type: ignore[attr-defined] + raise RuntimeError("kaboom") + return (True, [], False) + + calls = _wire(monkeypatch, indexing=[bad, good], failed=[], state=_state) + db = SimpleNamespace(rollback=lambda: calls.setdefault("rolled_back", True)) + provision.finalize_ready_onboarding_requests(db) # type: ignore[arg-type] + assert calls["finalized"] == [6] # good one still finalized + assert calls.get("rolled_back") is True # bad one rolled back diff --git a/backend/tests/unit/danswer/onboarding/test_notify.py b/backend/tests/unit/danswer/onboarding/test_notify.py new file mode 100644 index 00000000000..176ceb0c6d0 --- /dev/null +++ b/backend/tests/unit/danswer/onboarding/test_notify.py @@ -0,0 +1,209 @@ +"""Unit tests for onboarding Slack notifications (mocked — no network).""" +from types import SimpleNamespace + +import pytest + +from danswer.onboarding import notify + + +def _req() -> SimpleNamespace: + return SimpleNamespace( + id=7, + requester_email="jo@uipath.com", + payload={ + "team_name": "Automation Suite", + "channel": {"channel_name": "help-automation-suite"}, + "sources": [{"type": "web", "label": "Docs (cloud)"}], + }, + ) + + +def _patch_client(monkeypatch: pytest.MonkeyPatch, client_cls: type) -> None: + monkeypatch.setattr(notify, "WebClient", client_cls) + monkeypatch.setattr(notify, "fetch_tokens", lambda: SimpleNamespace(bot_token="x")) + + +def test_notify_posts_message(monkeypatch: pytest.MonkeyPatch) -> None: + captured: dict = {} + + class _Client: + def __init__(self, token: str) -> None: + pass + + def chat_postMessage(self, **kwargs: object) -> None: + captured.update(kwargs) + + _patch_client(monkeypatch, _Client) + monkeypatch.setattr(notify, "ONBOARDING_NOTIFY_CHANNEL", "darwin-devs") + notify.notify_onboarding_submitted(_req()) # type: ignore[arg-type] + + assert captured["channel"] == "darwin-devs" + text = captured["text"] + assert "jo@uipath.com" in text + assert "Automation Suite" in text + assert "help-automation-suite" in text + # Submitted notification links to the request's form so the admin can open, + # review and approve it directly. + assert "/admin/onboarding/7" in text + + +def test_notify_complete_includes_status_url( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: dict = {} + + class _Client: + def __init__(self, token: str) -> None: + pass + + def chat_postMessage(self, **kwargs: object) -> None: + captured.update(kwargs) + + _patch_client(monkeypatch, _Client) + monkeypatch.setattr(notify, "ONBOARDING_NOTIFY_CHANNEL", "darwin-devs") + monkeypatch.setattr(notify, "WEB_DOMAIN", "https://darwin.example.com") + notify.notify_onboarding_complete(_req()) # type: ignore[arg-type] + + text = captured["text"] + assert "is now live in #help-automation-suite" in text + assert "https://darwin.example.com/admin/onboarding" in text + + +def test_notify_swallows_slack_errors(monkeypatch: pytest.MonkeyPatch) -> None: + class _Client: + def __init__(self, token: str) -> None: + pass + + def chat_postMessage(self, **kwargs: object) -> None: + raise Exception("slack is down") + + _patch_client(monkeypatch, _Client) + monkeypatch.setattr(notify, "ONBOARDING_NOTIFY_CHANNEL", "darwin-devs") + # Must not raise — submission must never fail on a notification hiccup. + notify.notify_onboarding_submitted(_req()) # type: ignore[arg-type] + + +def test_notify_noop_when_channel_unset(monkeypatch: pytest.MonkeyPatch) -> None: + created = {"n": 0} + + class _Client: + def __init__(self, token: str) -> None: + created["n"] += 1 + + _patch_client(monkeypatch, _Client) + monkeypatch.setattr(notify, "ONBOARDING_NOTIFY_CHANNEL", "") + notify.notify_onboarding_submitted(_req()) # type: ignore[arg-type] + assert created["n"] == 0 # no client built, nothing posted + + +# --- customer-facing thread ------------------------------------------------ + + +def test_client_submitted_returns_ts_mentions_and_links( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: dict = {} + + class _Client: + def __init__(self, token: str) -> None: + pass + + def conversations_join(self, **kwargs: object) -> None: + captured["joined"] = kwargs.get("channel") + + def users_lookupByEmail(self, email: str) -> dict: + return {"user": {"id": "U999"}} + + def chat_postMessage(self, **kwargs: object) -> dict: + captured.update(kwargs) + return {"ts": "111.222"} + + _patch_client(monkeypatch, _Client) + monkeypatch.setattr(notify, "ONBOARDING_CLIENT_CHANNEL", "help-darwin") + monkeypatch.setattr(notify, "WEB_DOMAIN", "https://darwin.example.com") + + ts = notify.notify_client_submitted(_req()) # type: ignore[arg-type] + assert ts == "111.222" + assert captured["channel"] == "help-darwin" + assert "<@U999>" in captured["text"] # requester @mentioned + assert "https://darwin.example.com/onboarding?view=requests" in captured["text"] + assert "thread_ts" not in captured # root message, not a reply + + +def test_client_submitted_mention_falls_back_to_email( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: dict = {} + + class _Client: + def __init__(self, token: str) -> None: + pass + + def conversations_join(self, **kwargs: object) -> None: + pass + + def users_lookupByEmail(self, email: str) -> dict: + raise Exception("users_not_found") + + def chat_postMessage(self, **kwargs: object) -> dict: + captured.update(kwargs) + return {"ts": "1"} + + _patch_client(monkeypatch, _Client) + monkeypatch.setattr(notify, "ONBOARDING_CLIENT_CHANNEL", "help-darwin") + notify.notify_client_submitted(_req()) # type: ignore[arg-type] + assert "jo@uipath.com" in captured["text"] # plain-text fallback + + +def test_client_reply_skips_without_thread_ts( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # No stored thread ts -> must NOT post anything (never a top-level message). + built = {"n": 0} + + class _Client: + def __init__(self, token: str) -> None: + built["n"] += 1 + + def chat_postMessage(self, **kwargs: object) -> dict: + built["n"] += 100 + return {"ts": "x"} + + _patch_client(monkeypatch, _Client) + monkeypatch.setattr(notify, "ONBOARDING_CLIENT_CHANNEL", "help-darwin") + notify.notify_client_complete(_req()) # type: ignore[arg-type] + assert built["n"] == 0 + + +def test_client_reply_threads_when_ts_present( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: dict = {} + + class _Client: + def __init__(self, token: str) -> None: + pass + + def chat_postMessage(self, **kwargs: object) -> dict: + captured.update(kwargs) + return {"ts": "y"} + + _patch_client(monkeypatch, _Client) + monkeypatch.setattr(notify, "ONBOARDING_CLIENT_CHANNEL", "help-darwin") + req = _req() + req.help_thread_ts = "111.222" + notify.notify_client_complete(req) # type: ignore[arg-type] + assert captured["thread_ts"] == "111.222" # posted as a reply + assert "live in #help-automation-suite" in captured["text"] + + +def test_resolve_channel_accepts_id_name_or_link() -> None: + assert notify._resolve_channel("C07B2V8E99S") == "C07B2V8E99S" + assert notify._resolve_channel("darwin-devs") == "darwin-devs" + assert ( + notify._resolve_channel( + "https://uipath.enterprise.slack.com/archives/C07B2V8E99S" + ) + == "C07B2V8E99S" + ) + assert notify._resolve_channel(" #ops ") == "#ops" diff --git a/backend/tests/unit/danswer/onboarding/test_provision.py b/backend/tests/unit/danswer/onboarding/test_provision.py new file mode 100644 index 00000000000..9e8eb284864 --- /dev/null +++ b/backend/tests/unit/danswer/onboarding/test_provision.py @@ -0,0 +1,438 @@ +"""Unit tests for the onboarding provisioning builders (pure logic — no DB/IO).""" +import pytest + +from danswer.configs.constants import DocumentSource +from danswer.onboarding import provision +from danswer.onboarding.provision import _build_channel_config +from danswer.onboarding.provision import _build_connector_base +from danswer.onboarding.provision import _parse_github_repo +from danswer.onboarding.provision import _source_type_value + + +def _allow_public_dns(monkeypatch: pytest.MonkeyPatch) -> None: + """Make the web SSRF guard see any host as a public address, offline.""" + monkeypatch.setattr( + provision.socket, + "getaddrinfo", + lambda host, port: [(2, 1, 6, "", ("93.184.216.34", 0))], + ) + + +def test_web_docs_uipath_enables_all_versions( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _allow_public_dns(monkeypatch) + cb = _build_connector_base( + { + "type": "web", + "value": "https://docs.uipath.com/orchestrator/automation-cloud/latest", + }, + db_session=None, # type: ignore[arg-type] # web path doesn't touch the db + ) + assert cb.source == DocumentSource.WEB + cfg = cb.connector_specific_config + assert cfg["base_url"].startswith("https://docs.uipath.com/orchestrator") + assert cfg["uipath_latest_versions"] is True + assert cfg["max_versions"] == 3 # latest 3 versions + assert cfg["web_connector_type"] == "recursive" + + +def test_web_non_docs_does_not_enable_all_versions( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _allow_public_dns(monkeypatch) + cb = _build_connector_base( + {"type": "web", "value": "https://example.com/help"}, + db_session=None, # type: ignore[arg-type] + ) + assert "uipath_latest_versions" not in cb.connector_specific_config + + +@pytest.mark.parametrize( + "url", + [ + "http://127.0.0.1/internal", # loopback + "http://10.0.0.5/admin", # RFC1918 private + "http://169.254.169.254/latest/meta-data/", # cloud metadata (link-local) + "file:///etc/passwd", # non-http scheme + ], +) +def test_web_rejects_internal_targets(url: str) -> None: + # IP-literal / bad-scheme rejects need no DNS, so no monkeypatch required. + with pytest.raises(ValueError): + _build_connector_base( + {"type": "web", "value": url}, db_session=None # type: ignore[arg-type] + ) + + +def test_confluence_source_config(monkeypatch: pytest.MonkeyPatch) -> None: + url = "https://uipath.atlassian.net/wiki/spaces/DEV/overview" + monkeypatch.setattr( + provision, "_allowed_confluence_hosts", lambda db: {"uipath.atlassian.net"} + ) + cb = _build_connector_base( + {"type": "confluence", "value": url}, db_session=None # type: ignore[arg-type] + ) + assert cb.source == DocumentSource.CONFLUENCE + assert cb.connector_specific_config["wiki_page_url"] == url + + +def test_confluence_rejects_unknown_host(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + provision, "_allowed_confluence_hosts", lambda db: {"uipath.atlassian.net"} + ) + with pytest.raises(ValueError): + _build_connector_base( + {"type": "confluence", "value": "https://evil.example.com/wiki/spaces/X"}, + db_session=None, # type: ignore[arg-type] + ) + + +def test_confluence_rejects_when_no_known_hosts( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # No existing Confluence connector to vet against -> fail closed. + monkeypatch.setattr(provision, "_allowed_confluence_hosts", lambda db: set()) + with pytest.raises(ValueError): + _build_connector_base( + { + "type": "confluence", + "value": "https://uipath.atlassian.net/wiki/spaces/X", + }, + db_session=None, # type: ignore[arg-type] + ) + + +def test_github_source_config_parses_owner_repo() -> None: + cb = _build_connector_base( + {"type": "github", "value": "https://github.com/UiPath/testsfagents"}, + db_session=None, # type: ignore[arg-type] + ) + assert cb.source == DocumentSource.GITHUB + assert cb.connector_specific_config["repo_owner"] == "UiPath" + assert cb.connector_specific_config["repo_name"] == "testsfagents" + + +def test_github_rejects_non_github_host() -> None: + with pytest.raises(ValueError): + _build_connector_base( + {"type": "github", "value": "https://evil.example.com/UiPath/x"}, + db_session=None, # type: ignore[arg-type] + ) + + +# --- provision-time credential-access preflight --------------------------- + +from danswer.onboarding.provision import _assert_source_accessible # noqa: E402 +from danswer.onboarding.validation import ValidationResult # noqa: E402 + + +def test_preflight_web_is_noop() -> None: + # Web is public (no credential) — preflight must not raise. + _assert_source_accessible( + {"type": "web", "value": "https://example.com"}, db_session=None # type: ignore[arg-type] + ) + + +def test_preflight_raises_when_inaccessible(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + provision, + "validate_github_repo", + lambda value, db: ValidationResult(valid=False, message="no access"), + ) + with pytest.raises(ValueError, match="not accessible"): + _assert_source_accessible( + {"type": "github", "value": "https://github.com/UiPath/x"}, + db_session=None, # type: ignore[arg-type] + ) + + +def test_preflight_passes_when_accessible(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + provision, + "validate_slack_channel", + lambda value: ValidationResult(valid=True, message="#ok"), + ) + _assert_source_accessible( + {"type": "slack", "value": "help-x"}, db_session=None # type: ignore[arg-type] + ) + + +def test_parse_github_repo() -> None: + assert _parse_github_repo("https://github.com/UiPath/danswer") == ( + "UiPath", + "danswer", + ) + + +def test_source_type_value_mapping() -> None: + assert _source_type_value("web") == DocumentSource.WEB.value + assert _source_type_value("confluence") == DocumentSource.CONFLUENCE.value + assert _source_type_value("github") == DocumentSource.GITHUB.value + assert _source_type_value("slack") == DocumentSource.SLACK.value + + +def _payload(**over: object) -> dict: + base = { + "team_name": "Team X", + "channel": {"channel_id": "C1", "channel_name": "help-x"}, + "respond_tag_only": True, + "sme": {"enabled": False, "group_name": ""}, + "oncall": {"enabled": False, "schedule": ""}, + "jira": {"enabled": False}, + } + base.update(over) + return base + + +def test_channel_config_basic() -> None: + cfg = _build_channel_config(_payload(), prioritized_sources=["confluence", "web"]) + assert cfg["channel_names"] == ["help-x"] + assert cfg["respond_tag_only"] is True + assert cfg["prioritized_sources"] == ["confluence", "web"] + assert "enable_sme_validation" not in cfg + assert "opsgenie_schedule" not in cfg + assert "jira_config" not in cfg + + +def test_channel_config_all_options() -> None: + cfg = _build_channel_config( + _payload( + sme={"enabled": True, "group_name": "as-smes"}, + oncall={"enabled": True, "schedule": "AS-OnCall"}, + jira={ + "enabled": True, + "project_key": "AS", + "issue_type": "Bug", + "component": "core", + }, + ), + prioritized_sources=["web"], + ) + assert cfg["enable_sme_validation"] is True + assert cfg["sme_group_name"] == "as-smes" + assert cfg["opsgenie_schedule"] == "AS-OnCall" + assert cfg["jira_config"]["enable_jira_integration"] is True + assert cfg["jira_config"]["project_key"] == "AS" + + +def test_channel_config_oncall_enabled_but_no_schedule_omitted() -> None: + # Enabling on-call without a schedule shouldn't write an empty opsgenie value. + cfg = _build_channel_config( + _payload(oncall={"enabled": True, "schedule": ""}), + prioritized_sources=[], + ) + assert "opsgenie_schedule" not in cfg + + +# --- per-source refresh cadence ------------------------------------------- + +from danswer.onboarding.provision import _MONTHLY_REFRESH_FREQ # noqa: E402 +from danswer.onboarding.provision import DEFAULT_REFRESH_FREQ # noqa: E402 +from danswer.onboarding.provision import _dedup_confluence_sources # noqa: E402 +from danswer.onboarding.provision import _prioritized_sources # noqa: E402 + + +def test_web_source_refresh_is_monthly(monkeypatch: pytest.MonkeyPatch) -> None: + _allow_public_dns(monkeypatch) + cb = _build_connector_base( + {"type": "web", "value": "https://docs.uipath.com/x/latest"}, + db_session=None, # type: ignore[arg-type] + ) + assert cb.refresh_freq == _MONTHLY_REFRESH_FREQ + + +def test_confluence_source_refresh_is_daily(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + provision, "_allowed_confluence_hosts", lambda db: {"x.atlassian.net"} + ) + cb = _build_connector_base( + {"type": "confluence", "value": "https://x.atlassian.net/wiki/spaces/DEV/x"}, + db_session=None, # type: ignore[arg-type] + ) + assert cb.refresh_freq == DEFAULT_REFRESH_FREQ + + +# --- prioritized sources --------------------------------------------------- + + +def test_prioritized_sources_distinct_in_order() -> None: + sources = [ + {"type": "web", "value": "a"}, + {"type": "slack", "value": "b"}, + {"type": "web", "value": "c"}, # duplicate type + ] + assert _prioritized_sources(sources) == ["web", "slack"] + + +# --- confluence parent/child dedup ---------------------------------------- + + +def test_dedup_passthrough_when_fewer_than_two_confluence() -> None: + sources = [ + {"type": "confluence", "value": "https://x.atlassian.net/wiki/spaces/DEV/x"}, + {"type": "web", "value": "https://docs.example.com"}, + ] + assert _dedup_confluence_sources(sources, db_session=None) == sources # type: ignore[arg-type] + + +def test_dedup_space_supersedes_page() -> None: + space = { + "type": "confluence", + "value": "https://x.atlassian.net/wiki/spaces/DEV/overview", + } + page = { + "type": "confluence", + "value": "https://x.atlassian.net/wiki/spaces/DEV/pages/123/Title", + } + out = _dedup_confluence_sources([space, page], db_session=None) # type: ignore[arg-type] + assert out == [space] # the page (child of the space) is dropped + + +def test_dedup_parent_page_supersedes_child(monkeypatch: pytest.MonkeyPatch) -> None: + parent = { + "type": "confluence", + "value": "https://x.atlassian.net/wiki/spaces/DEV/pages/100/Parent", + } + child = { + "type": "confluence", + "value": "https://x.atlassian.net/wiki/spaces/DEV/pages/200/Child", + } + # 200's ancestor is 100 (also provided) -> child dropped. + monkeypatch.setattr( + provision, + "confluence_page_ancestors", + lambda ids, db: {"100": [], "200": ["100"]}, + ) + out = _dedup_confluence_sources([parent, child], db_session=None) # type: ignore[arg-type] + assert out == [parent] + + +def test_dedup_keeps_unrelated_pages(monkeypatch: pytest.MonkeyPatch) -> None: + a = { + "type": "confluence", + "value": "https://x.atlassian.net/wiki/spaces/DEV/pages/100/A", + } + b = { + "type": "confluence", + "value": "https://x.atlassian.net/wiki/spaces/DEV/pages/300/B", + } + monkeypatch.setattr( + provision, "confluence_page_ancestors", lambda ids, db: {"100": [], "300": []} + ) + out = _dedup_confluence_sources([a, b], db_session=None) # type: ignore[arg-type] + assert out == [a, b] + + +# --- provision_onboarding: cc_pair_ids records real cc_pair ids ------------ + +from types import SimpleNamespace # noqa: E402 + +from danswer.db.models import OnboardingStatus # noqa: E402 +from danswer.onboarding.provision import provision_onboarding # noqa: E402 + + +def test_provision_records_cc_pair_ids_not_connector_ids( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Regression: in this fork add_credential_to_connector returns + data=connector_id (not the cc_pair id). provision_onboarding must store the + real cc_pair ids on the request — otherwise the finalizer/status track the + wrong pairs (the bug that surfaced an unrelated '[local-copy] jira' and + dropped a real source). We simulate connector_id != cc_pair_id and assert the + request gets the cc_pair ids.""" + # Each created connector gets an incrementing id (1, 2, ...). + connector_ids = iter(range(1, 100)) + captured: dict = {} + + monkeypatch.setattr(provision, "update_onboarding_status", lambda *a, **k: None) + monkeypatch.setattr( + provision, "get_current_db_embedding_model", lambda db: SimpleNamespace(id=1) + ) + monkeypatch.setattr( + provision, "_dedup_confluence_sources", lambda sources, db: sources + ) + monkeypatch.setattr(provision, "_assert_source_accessible", lambda s, db: None) + monkeypatch.setattr(provision, "_build_connector_base", lambda s, db: s) + monkeypatch.setattr( + provision, + "create_connector", + lambda base, db: SimpleNamespace(id=next(connector_ids)), + ) + monkeypatch.setattr(provision, "_credential_id_for_source", lambda stype, db: 500) + # The fork's real return shape: .data is the CONNECTOR id. + monkeypatch.setattr( + provision, + "add_credential_to_connector", + lambda **kw: SimpleNamespace(data=kw["connector_id"]), + ) + # The real cc_pair id is deliberately different (connector_id + 1000). + monkeypatch.setattr( + provision, + "get_connector_credential_pair", + lambda connector_id, credential_id, db: SimpleNamespace(id=connector_id + 1000), + ) + monkeypatch.setattr( + provision, + "set_provisioned_ids", + lambda db, req, cc_pair_ids: captured.__setitem__("cc_pair_ids", cc_pair_ids), + ) + monkeypatch.setattr(provision, "create_index_attempt", lambda **k: 0) + + request = SimpleNamespace( + id=1, + payload={ + "team_name": "T", + "sources": [ + {"type": "web", "value": "https://a", "label": "A"}, + {"type": "slack", "value": "chan", "label": "B"}, + ], + }, + ) + provision_onboarding(request, SimpleNamespace(id=42), None) # type: ignore[arg-type] + + # connectors were 1 and 2; real cc_pair ids are 1001 and 1002 — NOT [1, 2]. + assert captured["cc_pair_ids"] == [1001, 1002] + + +def test_provision_marks_failed_when_cc_pair_missing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """If the freshly-created pair can't be found, provisioning fails loudly + (marks FAILED + raises) rather than silently recording a bad id.""" + statuses: list = [] + monkeypatch.setattr( + provision, + "update_onboarding_status", + lambda db, req, status, **kw: statuses.append(status), + ) + monkeypatch.setattr( + provision, "get_current_db_embedding_model", lambda db: SimpleNamespace(id=1) + ) + monkeypatch.setattr( + provision, "_dedup_confluence_sources", lambda sources, db: sources + ) + monkeypatch.setattr(provision, "_assert_source_accessible", lambda s, db: None) + monkeypatch.setattr(provision, "_build_connector_base", lambda s, db: s) + monkeypatch.setattr( + provision, "create_connector", lambda base, db: SimpleNamespace(id=7) + ) + monkeypatch.setattr(provision, "_credential_id_for_source", lambda stype, db: 500) + monkeypatch.setattr( + provision, "add_credential_to_connector", lambda **kw: SimpleNamespace(data=7) + ) + monkeypatch.setattr( + provision, "get_connector_credential_pair", lambda c, cr, db: None + ) + monkeypatch.setattr(provision, "set_provisioned_ids", lambda *a, **k: None) + + request = SimpleNamespace( + id=1, + payload={ + "team_name": "T", + "sources": [{"type": "web", "value": "https://a", "label": "A"}], + }, + ) + with pytest.raises(ValueError): + provision_onboarding(request, SimpleNamespace(id=42), None) # type: ignore[arg-type] + assert OnboardingStatus.FAILED in statuses diff --git a/backend/tests/unit/danswer/onboarding/test_validation.py b/backend/tests/unit/danswer/onboarding/test_validation.py new file mode 100644 index 00000000000..cb93a581642 --- /dev/null +++ b/backend/tests/unit/danswer/onboarding/test_validation.py @@ -0,0 +1,481 @@ +"""Unit tests for onboarding docs-URL validation (pure — no network). + +Slack/Confluence validators hit live APIs and are covered by manual/integration +checks, not here.""" +import pytest + +from danswer.onboarding import validation +from danswer.onboarding.validation import validate_docs_url +from danswer.onboarding.validation import validate_github_repo +from danswer.onboarding.validation import validate_jira_filter + + +class _Cred: + def __init__(self, cj: dict) -> None: + self.credential_json = cj + + +class _Scalars: + def __init__(self, creds: list) -> None: + self._creds = creds + + def scalars(self): # type: ignore[no-untyped-def] + return iter(self._creds) + + +class _FakeDb: + """Minimal Session stand-in for `_first_github_token`'s single query.""" + + def __init__(self, creds: list) -> None: + self._creds = creds + + def execute(self, *args, **kwargs): # type: ignore[no-untyped-def] + return _Scalars(self._creds) + + +def test_docs_url_valid_root_normalizes() -> None: + r = validate_docs_url( + "https://docs.uipath.com/orchestrator/automation-cloud/latest/user-guide/x" + ) + assert r.valid + # Normalized to the product root (version/'latest' segment stripped). + assert ( + r.resolved["root_url"] + == "https://docs.uipath.com/orchestrator/automation-cloud" + ) + + +def test_docs_url_rejects_non_docs_domain() -> None: + r = validate_docs_url("https://example.com/foo") + assert not r.valid + + +def test_docs_url_requires_product_path() -> None: + r = validate_docs_url("https://docs.uipath.com/") + assert not r.valid + + +def test_docs_url_empty() -> None: + assert not validate_docs_url("").valid + + +# --- validate_github_repo ------------------------------------------------- + + +@pytest.mark.parametrize( + "url", + [ + "", # empty + "https://gitlab.com/UiPath/x", # wrong host + "https://evil.example.com/UiPath/x", # non-github host + "https://github.com/UiPath", # missing repo segment + ], +) +def test_github_repo_rejects_bad_input(url: str) -> None: + # These all return before touching the DB, so db_session can be None. + assert not validate_github_repo(url, db_session=None).valid # type: ignore[arg-type] + + +def test_github_repo_no_credential_is_unverified() -> None: + # No GitHub token on file -> valid but explicitly "unverified". + r = validate_github_repo("https://github.com/UiPath/danswer", _FakeDb([])) # type: ignore[arg-type] + assert r.valid + assert "unverified" in r.message.lower() + assert r.resolved == {"repo_owner": "UiPath", "repo_name": "danswer"} + + +def test_github_repo_accessible(monkeypatch: pytest.MonkeyPatch) -> None: + import github as github_mod + + class _OkGithub: + def __init__(self, token: str, base_url: str | None = None) -> None: + pass + + def get_repo(self, full_name: str) -> object: + return object() + + monkeypatch.setattr(github_mod, "Github", _OkGithub) + db = _FakeDb([_Cred({"github_access_token": "ghp_fake"})]) + r = validate_github_repo("https://github.com/UiPath/danswer.git", db) # type: ignore[arg-type] + assert r.valid + assert r.message == "UiPath/danswer" # .git suffix stripped + + +def test_github_repo_inaccessible_reports_generic_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import github as github_mod + + class _DenyGithub: + def __init__(self, token: str, base_url: str | None = None) -> None: + pass + + def get_repo(self, full_name: str) -> object: + # Message intentionally contains a token-like string to prove it is + # NOT propagated into the ValidationResult. + raise Exception("404 {'token': 'ghp_should_not_leak'}") + + monkeypatch.setattr(github_mod, "Github", _DenyGithub) + db = _FakeDb([_Cred({"github_access_token": "ghp_fake"})]) + r = validate_github_repo("https://github.com/UiPath/secret-repo", db) # type: ignore[arg-type] + assert not r.valid + assert "not found or the GitHub app lacks access" in r.message + assert "ghp_" not in r.message # no token / raw payload leakage + + +# --- validate_jira_filter ------------------------------------------------- + + +def test_jira_filter_empty() -> None: + assert not validate_jira_filter("", db_session=None).valid # type: ignore[arg-type] + + +def test_jira_filter_unverified_without_connector( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(validation, "jira_base_url", lambda db: None) + monkeypatch.setattr(validation, "_first_jira_credential", lambda db: None) + r = validate_jira_filter("project = ABC", db_session=None) # type: ignore[arg-type] + assert r.valid + assert "unverified" in r.message.lower() + + +def test_jira_filter_valid(monkeypatch: pytest.MonkeyPatch) -> None: + import jira as jira_mod + + class _OkJira: + def __init__(self, **kwargs: object) -> None: + pass + + def search_issues(self, jql: str, maxResults: int = 1) -> list: + return [] + + monkeypatch.setattr( + validation, "jira_base_url", lambda db: "https://x.atlassian.net" + ) + monkeypatch.setattr( + validation, + "_first_jira_credential", + lambda db: {"jira_api_token": "t", "jira_user_email": "e@x.com"}, + ) + monkeypatch.setattr(jira_mod, "JIRA", _OkJira) + r = validate_jira_filter("project = ABC", db_session=None) # type: ignore[arg-type] + assert r.valid + assert r.message == "Jira filter OK" + + +def test_jira_filter_invalid_reports_generic_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import jira as jira_mod + + class _BadJira: + def __init__(self, **kwargs: object) -> None: + pass + + def search_issues(self, jql: str, maxResults: int = 1) -> list: + raise Exception("400 bad JQL token=ghp_should_not_leak") + + monkeypatch.setattr( + validation, "jira_base_url", lambda db: "https://x.atlassian.net" + ) + monkeypatch.setattr( + validation, "_first_jira_credential", lambda db: {"jira_api_token": "t"} + ) + monkeypatch.setattr(jira_mod, "JIRA", _BadJira) + r = validate_jira_filter("nonsense jql", db_session=None) # type: ignore[arg-type] + assert not r.valid + assert "Invalid Jira filter" in r.message + assert "token=" not in r.message # no payload/token leakage + + +# --- validate_slack_channel (mocked bot client) --------------------------- +from slack_sdk.errors import SlackApiError # noqa: E402 + +from danswer.onboarding.validation import validate_slack_channel # noqa: E402 +from danswer.onboarding.validation import validate_slack_group # noqa: E402 +from danswer.onboarding.validation import validate_confluence_url # noqa: E402 + + +class _SlackClient: + """Fake WebClient covering conversations_info + users_conversations paging.""" + + def __init__( + self, + info: dict | None = None, + info_error: str | None = None, + member_channels: list | None = None, + ): + self._info = info + self._info_error = info_error + self._member = member_channels or [] + + def conversations_info(self, channel: str) -> dict: + if self._info_error: + raise SlackApiError(self._info_error, {"error": self._info_error}) + return {"channel": self._info} + + def users_conversations( + self, limit: int = 1000, cursor: str | None = None, **kw: object + ) -> dict: + return {"channels": self._member, "response_metadata": {"next_cursor": ""}} + + +def _patch_bot(monkeypatch: pytest.MonkeyPatch, client: object) -> None: + monkeypatch.setattr(validation, "_bot_client", lambda: client) + + +def test_slack_channel_empty() -> None: + assert not validate_slack_channel("").valid + + +def test_slack_channel_from_link(monkeypatch: pytest.MonkeyPatch) -> None: + _patch_bot(monkeypatch, _SlackClient(info={"id": "C0ABC123", "name": "help-team"})) + r = validate_slack_channel( + "https://uipath-product.slack.com/archives/C0ABC123/p1712" + ) + assert r.valid + assert r.resolved["channel_id"] == "C0ABC123" + assert r.resolved["channel_name"] == "help-team" + + +def test_slack_channel_from_mention(monkeypatch: pytest.MonkeyPatch) -> None: + _patch_bot(monkeypatch, _SlackClient(info={"id": "C0ABC123", "name": "help-team"})) + assert validate_slack_channel("<#C0ABC123|help-team>").valid + + +def test_slack_channel_link_member(monkeypatch: pytest.MonkeyPatch) -> None: + _patch_bot( + monkeypatch, + _SlackClient(info={"id": "C1", "name": "help-team", "is_member": True}), + ) + r = validate_slack_channel("https://x.slack.com/archives/C1") + assert r.valid + assert "already in this channel" in r.message + + +def test_slack_channel_link_public_not_member( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _patch_bot( + monkeypatch, + _SlackClient( + info={ + "id": "C1", + "name": "help-team", + "is_member": False, + "is_private": False, + } + ), + ) + r = validate_slack_channel("https://x.slack.com/archives/C1") + assert r.valid + assert "auto-join" in r.message # public -> connector self-joins + + +def test_slack_channel_link_private_not_member( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _patch_bot( + monkeypatch, + _SlackClient( + info={ + "id": "C1", + "name": "secret", + "is_member": False, + "is_private": True, + } + ), + ) + r = validate_slack_channel("https://x.slack.com/archives/C1") + assert r.valid + assert "invite" in r.message.lower() + assert "private" in r.message.lower() # private -> must be invited + + +def test_slack_channel_bad_id_rejected(monkeypatch: pytest.MonkeyPatch) -> None: + _patch_bot(monkeypatch, _SlackClient(info_error="channel_not_found")) + r = validate_slack_channel("C0DEADBEEF") + assert not r.valid + assert "not found" in r.message.lower() + + +def test_slack_channel_bare_name_member(monkeypatch: pytest.MonkeyPatch) -> None: + _patch_bot( + monkeypatch, + _SlackClient(member_channels=[{"id": "C1", "name": "help-team"}]), + ) + r = validate_slack_channel("help-team") + assert r.valid + assert "already in this channel" in r.message + assert r.resolved["channel_id"] == "C1" + + +def test_slack_channel_bare_name_not_member_accepted( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Not in the bot's channels yet -> still accepted (bot added during onboarding). + _patch_bot(monkeypatch, _SlackClient(member_channels=[])) + r = validate_slack_channel("brand-new-channel") + assert r.valid + assert "auto-join" in r.message # public auto-join; private needs an invite + assert r.resolved["channel_name"] == "brand-new-channel" + + +# --- validate_slack_group (comma-separated handles) ----------------------- + + +def test_slack_group_empty() -> None: + assert not validate_slack_group("").valid + + +def test_slack_group_all_found(monkeypatch: pytest.MonkeyPatch) -> None: + _patch_bot(monkeypatch, object()) + monkeypatch.setattr( + validation, + "fetch_groupids_from_names", + lambda handles, client: (["G1", "G2"], []), + ) + r = validate_slack_group("@as-dri, @sre-dri") + assert r.valid + assert r.message == "@as-dri, @sre-dri" + + +def test_slack_group_some_missing(monkeypatch: pytest.MonkeyPatch) -> None: + _patch_bot(monkeypatch, object()) + monkeypatch.setattr( + validation, + "fetch_groupids_from_names", + lambda handles, client: (["G1"], ["sre-dri"]), + ) + r = validate_slack_group("@as-dri, @sre-dri") + assert not r.valid + assert "sre-dri" in r.message + + +# --- validate_confluence_url (mocked host allowlist + client) ------------- + + +def test_confluence_url_bad_host_rejected(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + validation, "_allowed_confluence_hosts", lambda db: {"uipath.atlassian.net"} + ) + r = validate_confluence_url("https://evil.atlassian.net/wiki/spaces/X", None) # type: ignore[arg-type] + assert not r.valid + assert "host not allowed" in r.message.lower() + + +def test_confluence_url_unverified_without_creds( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(validation, "_allowed_confluence_hosts", lambda db: set()) + monkeypatch.setattr(validation, "_first_confluence_credential", lambda db: None) + r = validate_confluence_url("https://uipath.atlassian.net/wiki/spaces/DEV/x", None) # type: ignore[arg-type] + assert r.valid + assert "unverified" in r.message.lower() + + +def test_confluence_url_space_accessible(monkeypatch: pytest.MonkeyPatch) -> None: + import atlassian as atlassian_mod + + class _OkConfluence: + def __init__(self, **kwargs: object) -> None: + pass + + def get_space(self, space: str) -> dict: + return {"key": space} + + monkeypatch.setattr( + validation, "_allowed_confluence_hosts", lambda db: {"uipath.atlassian.net"} + ) + monkeypatch.setattr( + validation, + "_first_confluence_credential", + lambda db: {"confluence_username": "u", "confluence_access_token": "t"}, + ) + monkeypatch.setattr(atlassian_mod, "Confluence", _OkConfluence) + r = validate_confluence_url("https://uipath.atlassian.net/wiki/spaces/DEV/x", None) # type: ignore[arg-type] + assert r.valid + assert r.resolved["space"] == "DEV" + assert "whole" in r.message and "space" in r.message # space url -> whole space + + +def test_confluence_url_page_scope(monkeypatch: pytest.MonkeyPatch) -> None: + import atlassian as atlassian_mod + + class _OkConfluence: + def __init__(self, **kwargs: object) -> None: + pass + + def get_page_by_id(self, page_id: str) -> dict: + return {"id": page_id} + + monkeypatch.setattr( + validation, "_allowed_confluence_hosts", lambda db: {"uipath.atlassian.net"} + ) + monkeypatch.setattr( + validation, + "_first_confluence_credential", + lambda db: {"confluence_username": "u", "confluence_access_token": "t"}, + ) + monkeypatch.setattr(atlassian_mod, "Confluence", _OkConfluence) + r = validate_confluence_url( + "https://uipath.atlassian.net/wiki/spaces/DEV/pages/88998707616/Pro+Dev", + None, # type: ignore[arg-type] + ) + assert r.valid + # page url -> page + children, NOT the whole space + assert "child pages" in r.message + assert "whole" not in r.message + assert r.resolved["page_id"] == "88998707616" + + +def test_confluence_url_page_inaccessible(monkeypatch: pytest.MonkeyPatch) -> None: + import atlassian as atlassian_mod + + class _DenyConfluence: + def __init__(self, **kwargs: object) -> None: + pass + + def get_page_by_id(self, page_id: str) -> dict: + raise Exception("403 forbidden token=secret") + + monkeypatch.setattr( + validation, "_allowed_confluence_hosts", lambda db: {"uipath.atlassian.net"} + ) + monkeypatch.setattr( + validation, + "_first_confluence_credential", + lambda db: {"confluence_username": "u", "confluence_access_token": "t"}, + ) + monkeypatch.setattr(atlassian_mod, "Confluence", _DenyConfluence) + r = validate_confluence_url( + "https://uipath.atlassian.net/wiki/spaces/DEV/pages/123/T", None # type: ignore[arg-type] + ) + assert not r.valid + assert "Page not found" in r.message + assert "secret" not in r.message # no token/exception leakage + + +def test_confluence_url_space_inaccessible(monkeypatch: pytest.MonkeyPatch) -> None: + import atlassian as atlassian_mod + + class _DenyConfluence: + def __init__(self, **kwargs: object) -> None: + pass + + def get_space(self, space: str) -> dict: + raise Exception("403 forbidden token=secret") + + monkeypatch.setattr( + validation, "_allowed_confluence_hosts", lambda db: {"uipath.atlassian.net"} + ) + monkeypatch.setattr( + validation, + "_first_confluence_credential", + lambda db: {"confluence_username": "u", "confluence_access_token": "t"}, + ) + monkeypatch.setattr(atlassian_mod, "Confluence", _DenyConfluence) + r = validate_confluence_url("https://uipath.atlassian.net/wiki/spaces/DEV/x", None) # type: ignore[arg-type] + assert not r.valid + assert "secret" not in r.message # no token/exception leakage diff --git a/k8s/overlays/prod/env.properties b/k8s/overlays/prod/env.properties index 95e21ec5cc0..4199b72a4e8 100644 --- a/k8s/overlays/prod/env.properties +++ b/k8s/overlays/prod/env.properties @@ -190,6 +190,13 @@ DANSWER_BOT_RESPOND_EVERY_CHANNEL= DANSWER_BOT_DISABLE_COT= NOTIFY_SLACKBOT_NO_ANSWER= +# --- Onboarding notifications --- +# Internal ops channel (admin review link + error details): #darwin-devs. +ONBOARDING_NOTIFY_CHANNEL=C07B2V8E99S +# Customer-facing thread (requester @mentioned; lifecycle updates as replies): +# #help-darwin. +ONBOARDING_CLIENT_CHANNEL=C077AFEGCKZ + # --- SMTP (non-secret bits) --- SMTP_SERVER= SMTP_PORT= diff --git a/k8s/overlays/prod/kustomization.yaml b/k8s/overlays/prod/kustomization.yaml index d04b06b4f26..51527829e37 100644 --- a/k8s/overlays/prod/kustomization.yaml +++ b/k8s/overlays/prod/kustomization.yaml @@ -30,10 +30,10 @@ namespace: darwin images: - name: danswer-backend newName: sfbrdevhelmweacr.azurecr.io/danswer/danswer-backend - newTag: vha-216 + newTag: vha-220 - name: danswer-web-server newName: sfbrdevhelmweacr.azurecr.io/danswer/danswer-web-server - newTag: vha-116 + newTag: vha-117 - name: danswer-model-server newName: danswer/danswer-model-server newTag: v0.3.94 diff --git a/web/src/app/admin/bot/SlackBotConfigCreationForm.tsx b/web/src/app/admin/bot/SlackBotConfigCreationForm.tsx index 1e8e9041848..7125c0fbf08 100644 --- a/web/src/app/admin/bot/SlackBotConfigCreationForm.tsx +++ b/web/src/app/admin/bot/SlackBotConfigCreationForm.tsx @@ -1,6 +1,7 @@ "use client"; import { ArrayHelpers, FieldArray, Form, Formik } from "formik"; +import { FiCheck } from "react-icons/fi"; import * as Yup from "yup"; import { usePopup } from "@/components/admin/connectors/Popup"; import { DocumentSet, SlackBotConfig } from "@/lib/types"; @@ -542,17 +543,19 @@ export const SlackBotCreationForm = ({ key={documentSet.id} className={ ` - px-3 + px-3 py-1 - rounded-lg + rounded-lg border - border-border - w-fit - flex - cursor-pointer ` + + w-fit + flex + items-center + gap-1.5 + cursor-pointer + transition-colors ` + (isSelected - ? " bg-hover" - : " bg-background hover:bg-hover-light") + ? " border-accent bg-accent/10 font-medium text-accent" + : " border-border bg-background text-default hover:bg-hover-light") } onClick={() => { if (isSelected) { @@ -562,6 +565,9 @@ export const SlackBotCreationForm = ({ } }} > + {isSelected && ( + + )}
{documentSet.name}
diff --git a/web/src/app/admin/onboarding/OnboardingRequestsTable.tsx b/web/src/app/admin/onboarding/OnboardingRequestsTable.tsx new file mode 100644 index 00000000000..853f506a56d --- /dev/null +++ b/web/src/app/admin/onboarding/OnboardingRequestsTable.tsx @@ -0,0 +1,314 @@ +"use client"; + +import { useEffect, useState } from "react"; +import Link from "next/link"; +import { + OnboardingRequestSnapshot, + OnboardingStatusResponse, +} from "@/lib/onboarding/interfaces"; +import { SourceStatusSummary } from "@/app/onboarding/SourceStatusSummary"; +import { OnboardingJourney } from "@/app/onboarding/OnboardingJourney"; + +type FilterKey = + "active" | "pending" | "in_progress" | "failed" | "complete" | "all"; + +const FILTERS: { + key: FilterKey; + label: string; + match: (s: string) => boolean; +}[] = [ + { + key: "active", + label: "Active", + // Anything not finished — needs attention. + match: (s) => !["complete", "rejected", "cancelled"].includes(s), + }, + { key: "pending", label: "Pending", match: (s) => s === "pending" }, + { + key: "in_progress", + label: "In progress", + match: (s) => ["provisioning", "indexing"].includes(s), + }, + { key: "failed", label: "Failed", match: (s) => s === "failed" }, + { key: "complete", label: "Complete", match: (s) => s === "complete" }, + { key: "all", label: "All", match: () => true }, +]; + +const PAGE_SIZE = 10; + +function Badge({ status }: { status: string }) { + const color = + status === "complete" + ? "text-link" + : status === "failed" || status === "rejected" + ? "text-error" + : status === "pending" + ? "text-default" + : "text-subtle"; + return ( + {status} + ); +} + +export function OnboardingRequestsTable() { + const [requests, setRequests] = useState([]); + const [statuses, setStatuses] = useState< + Record + >({}); + const [busy, setBusy] = useState(null); + const [error, setError] = useState(null); + // Default view hides finished requests (complete/rejected/cancelled) so the + // queue stays focused on what needs attention; switchable + paginated so the + // page stays usable well past 10+ requests. + const [filter, setFilter] = useState("active"); + const [page, setPage] = useState(0); + + async function refresh() { + const r = await fetch("/api/admin/onboarding"); + if (!r.ok) return; + const data = (await r.json()) as OnboardingRequestSnapshot[]; + setRequests(data); + // Auto-load per-source scrape status for provisioned requests so the journey + // + rollup populate without a manual "Refresh scrape status" click. + await Promise.all( + data + .filter((req) => (req.cc_pair_ids?.length ?? 0) > 0) + .map((req) => loadStatus(req.id)) + ); + } + + useEffect(() => { + void refresh(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + async function decide(id: number, action: "approve" | "reject") { + setError(null); + let reason: string | null = null; + if (action === "reject") { + reason = window.prompt("Reason for rejecting (optional)?") || null; + } + setBusy(id); + try { + const r = await fetch(`/api/admin/onboarding/${id}/${action}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(action === "reject" ? { reason } : {}), + }); + if (!r.ok) { + setError( + (await r.json().catch(() => null))?.detail || `Failed (${r.status}).` + ); + return; + } + await refresh(); + } finally { + setBusy(null); + } + } + + async function loadStatus(id: number) { + const r = await fetch(`/api/onboarding/${id}/status`); + if (r.ok) { + const data = (await r.json()) as OnboardingStatusResponse; + setStatuses((p) => ({ ...p, [id]: data })); + } + } + + if (requests.length === 0) { + return

No onboarding requests yet.

; + } + + const activeFilter = FILTERS.find((f) => f.key === filter) ?? FILTERS[0]; + const filtered = requests.filter((r) => activeFilter.match(r.status)); + const totalPages = Math.max(1, Math.ceil(filtered.length / PAGE_SIZE)); + const clampedPage = Math.min(page, totalPages - 1); + const pageItems = filtered.slice( + clampedPage * PAGE_SIZE, + clampedPage * PAGE_SIZE + PAGE_SIZE + ); + + return ( +
+ {error &&

{error}

} + + {/* Filter bar */} +
+ {FILTERS.map((f) => { + const count = requests.filter((r) => f.match(r.status)).length; + const selected = f.key === filter; + return ( + + ); + })} +
+ + {filtered.length === 0 ? ( +

+ No {activeFilter.label.toLowerCase()} onboarding requests. +

+ ) : ( + pageItems.map((req) => ( +
+
+
+
+ {req.payload.team_name} → #{req.payload.channel.channel_name} +
+
+ by {req.requester_email} ·{" "} + {new Date(req.created_at).toLocaleString()} +
+
+
+ + {statuses[req.id] && ( + + )} +
+
+ + + +
+ Sources:{" "} + {req.payload.sources + .map((s) => `${s.type}:${s.label || s.value}`) + .join(", ")} +
+
+ SME:{" "} + {req.payload.sme.enabled + ? req.payload.sme.group_name || "yes" + : "no"}{" "} + · On-call:{" "} + {req.payload.oncall.enabled + ? req.payload.oncall.schedule || "yes" + : "no"}{" "} + · Jira:{" "} + {req.payload.jira.enabled + ? req.payload.jira.project_key || "yes" + : "no"} +
+ {req.error_msg && ( +

{req.error_msg}

+ )} + + {req.status === "pending" && ( +
+ + Open / edit + + + +
+ )} + + {(req.cc_pair_ids?.length ?? 0) > 0 && ( +
+ + {statuses[req.id]?.sources.map((s) => ( +
+
+ + {s.name} + + + + + + {s.docs_indexed} docs + +
+ {s.error_msg && ( +

+ {s.error_msg} +

+ )} +
+ ))} +
+ )} +
+ )) + )} + + {totalPages > 1 && ( +
+ + Showing {clampedPage * PAGE_SIZE + 1}– + {Math.min((clampedPage + 1) * PAGE_SIZE, filtered.length)} of{" "} + {filtered.length} + +
+ + + Page {clampedPage + 1} / {totalPages} + + +
+
+ )} +
+ ); +} diff --git a/web/src/app/admin/onboarding/[id]/page.tsx b/web/src/app/admin/onboarding/[id]/page.tsx new file mode 100644 index 00000000000..5422307f307 --- /dev/null +++ b/web/src/app/admin/onboarding/[id]/page.tsx @@ -0,0 +1,48 @@ +"use client"; + +import { Suspense, useEffect, useState } from "react"; +import { OnboardingForm } from "@/app/onboarding/OnboardingForm"; +import { OnboardingRequestSnapshot } from "@/lib/onboarding/interfaces"; + +// Admin opens a pending request in the shared onboarding form (prefilled) to fix +// gaps and approve. Gated by the /admin layout (admin-only). +export default function Page({ params }: { params: { id: string } }) { + const [request, setRequest] = useState( + null + ); + const [error, setError] = useState(null); + + useEffect(() => { + fetch(`/api/admin/onboarding/${params.id}`) + .then(async (r) => { + if (!r.ok) { + setError(`Couldn't load request (${r.status}).`); + return; + } + setRequest((await r.json()) as OnboardingRequestSnapshot); + }) + .catch(() => setError("Couldn't load request.")); + }, [params.id]); + + if (error) { + return ( +
+ {error} +
+ ); + } + if (!request) { + return ( +
+ Loading… +
+ ); + } + return ( + // Suspense: OnboardingForm reads useSearchParams; the admin layout doesn't + // wrap children in one, so provide it here for the prod build. + + + + ); +} diff --git a/web/src/app/admin/onboarding/page.tsx b/web/src/app/admin/onboarding/page.tsx new file mode 100644 index 00000000000..e424a6e13df --- /dev/null +++ b/web/src/app/admin/onboarding/page.tsx @@ -0,0 +1,19 @@ +import { AdminPageTitle } from "@/components/admin/Title"; +import { RobotIcon } from "@/components/icons/icons"; +import { OnboardingRequestsTable } from "./OnboardingRequestsTable"; + +export default function Page() { + return ( +
+ } + title="Onboarding requests" + /> +

+ Approve a request to auto-provision the team's assistant, sources + (scraped with bumped priority), and Slack bot config. +

+ +
+ ); +} diff --git a/web/src/app/chat/ChatPage.tsx b/web/src/app/chat/ChatPage.tsx index da432589d2d..524a643fc5c 100644 --- a/web/src/app/chat/ChatPage.tsx +++ b/web/src/app/chat/ChatPage.tsx @@ -49,7 +49,8 @@ import { DocumentSidebar } from "./documentSidebar/DocumentSidebar"; import { DanswerInitializingLoader } from "@/components/DanswerInitializingLoader"; import { FeedbackModal } from "./modal/FeedbackModal"; import { ShareChatSessionModal } from "./modal/ShareChatSessionModal"; -import { FiArrowDown, FiShare2 } from "react-icons/fi"; +import { FiArrowDown, FiShare2, FiUserPlus } from "react-icons/fi"; +import Link from "next/link"; import { ChatIntro } from "./ChatIntro"; import { AIMessage, HumanMessage } from "./message/Messages"; import { ThreeDots } from "react-loader-spinner"; @@ -1278,6 +1279,20 @@ export function ChatPage({
+ + + + {chatSessionIdRef.current !== null && (
setSharingModalVisible(true)} @@ -1407,7 +1422,9 @@ export function ChatPage({ content={message.message} files={message.files} query={messageHistory[i]?.query || undefined} - personaName={assistantDisplayName(livePersona)} + personaName={assistantDisplayName( + livePersona + )} citedDocuments={getCitedDocumentsFromMessage( message )} @@ -1517,7 +1534,9 @@ export function ChatPage({ {message.message} diff --git a/web/src/app/chat/sessionSidebar/ChatSidebar.tsx b/web/src/app/chat/sessionSidebar/ChatSidebar.tsx index d3a7f8f6665..ee7981d8239 100644 --- a/web/src/app/chat/sessionSidebar/ChatSidebar.tsx +++ b/web/src/app/chat/sessionSidebar/ChatSidebar.tsx @@ -4,8 +4,10 @@ import { FiBook, FiEdit, FiFolderPlus, + FiList, FiLoader, FiPlusSquare, + FiUserPlus, } from "react-icons/fi"; import { useContext, useEffect, useRef, useState, useTransition } from "react"; import Link from "next/link"; @@ -191,6 +193,26 @@ export const ChatSidebar = ({
+
+ + +
+ Onboard a Team +
+ + +
+ +
+ + +
+ My onboarding requests +
+ + +
+
+ + + {text} + + + ); +} + +// --- validation result line ------------------------------------------------- + +function ResultLine({ result }: { result: ValidationResult | null }) { + if (!result) return null; + return ( +

+ {result.valid ? ( + + ) : ( + + )} + {result.message} +

+ ); +} + +// --- inline-validated text field -------------------------------------------- + +function ValidatedField({ + label, + placeholder, + kind, + value, + onChange, + onResolved, + optional, + info, + clientCheck, + helpText, +}: { + label: string; + placeholder?: string; + kind: ValidateKind; + value: string; + onChange: (v: string) => void; + onResolved?: (r: ValidationResult) => void; + optional?: boolean; + info?: string; + clientCheck?: ClientCheck; + helpText?: string; +}) { + const [result, setResult] = useState(null); + const [checking, setChecking] = useState(false); + // Keep onResolved fresh without retriggering the debounce effect. + const onResolvedRef = useRef(onResolved); + onResolvedRef.current = onResolved; + + const hint = value.trim() ? (clientCheck?.(value) ?? null) : null; + + // Debounced backend validation: fires VALIDATE_DEBOUNCE_MS after the last + // keystroke, and is skipped entirely while the local format check fails. + useEffect(() => { + if (!value.trim() || hint) { + setResult(null); + onResolvedRef.current?.({ valid: false, message: "", resolved: {} }); + return; + } + const t = setTimeout(async () => { + setChecking(true); + const r = await validateOnboardingField(kind, value); + setResult(r); + setChecking(false); + onResolvedRef.current?.(r); + }, VALIDATE_DEBOUNCE_MS); + return () => clearTimeout(t); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [value, kind, hint]); + + return ( +
+ + { + onChange(e.target.value); + setResult(null); + }} + className={inputClass} + /> + {helpText &&

{helpText}

} + {checking &&

Validating…

} + {!checking && result && } + {!checking && !result && hint && ( +

{hint}

+ )} +
+ ); +} + +// --- status badge ----------------------------------------------------------- + +function StatusBadge({ status }: { status: string }) { + const tone = + status === "complete" + ? "bg-link/10 text-link" + : status === "failed" || status === "rejected" + ? "bg-error/10 text-error" + : status === "cancelled" + ? "bg-subtle/10 text-subtle" + : "bg-accent/10 text-accent"; + return ( + + {status} + + ); +} + +// --- one extra source row --------------------------------------------------- + +const SOURCE_KIND: Record = { + web: "docs", + confluence: "confluence", + slack: "slack_channel", + jira: "jira", +}; + +const SOURCE_PLACEHOLDER: Record = { + web: "https://docs.example.com/…", + confluence: "https://.atlassian.net/wiki/spaces/KEY", + slack: "#another-channel", + jira: "project = ABC AND status != Done", +}; + +// Admin edit: rebuild the editable "additional sources" from a stored payload. +// Drop the auto-added channel-history slack source (re-derived from the channel); +// everything else (incl. docs) becomes an editable row. +function prefillExtraSources( + sources: OnboardingSource[], + channelName: string +): OnboardingSource[] { + return sources.filter( + (s) => + !( + s.type === "slack" && + (s.value === channelName || (s.label ?? "").endsWith(" history")) + ) + ); +} + +function SourceRow({ + source, + onChange, + onRemove, + onMove, + canMoveUp, + canMoveDown, +}: { + source: OnboardingSource; + onChange: (s: OnboardingSource) => void; + onRemove: () => void; + onMove: (dir: -1 | 1) => void; + canMoveUp: boolean; + canMoveDown: boolean; +}) { + const [result, setResult] = useState(null); + const [checking, setChecking] = useState(false); + const hint = source.value.trim() + ? SOURCE_CLIENT_CHECK[source.type](source.value) + : null; + + useEffect(() => { + if (!source.value.trim() || hint) { + setResult(null); + return; + } + const t = setTimeout(async () => { + setChecking(true); + setResult( + await validateOnboardingField(SOURCE_KIND[source.type], source.value) + ); + setChecking(false); + }, VALIDATE_DEBOUNCE_MS); + return () => clearTimeout(t); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [source.value, source.type, hint]); + + return ( +
+ +
+ { + onChange({ ...source, value: e.target.value }); + setResult(null); + }} + className={inputClass} + /> + {checking &&

Validating…

} + {!checking && result && } + {!checking && !result && hint && ( +

{hint}

+ )} +
+
+ + + +
+
+ ); +} + +// --- section shell with a numbered step marker ------------------------------ + +function Section({ + step, + title, + hint, + children, +}: { + step: number; + title: string; + hint?: string; + children: React.ReactNode; +}) { + return ( +
+
+ + {step} + +
+

{title}

+ {hint &&

{hint}

} +
+
+ {children} +
+ ); +} + +// --- main form -------------------------------------------------------------- + +export function OnboardingForm({ + initialRequest, + editMode = "admin", +}: { + // When set, the form runs in "edit a pending request" mode (prefilled). + initialRequest?: OnboardingRequestSnapshot; + // Who's editing: admin (save + approve) or the requester (save only). + editMode?: "admin" | "requester"; +} = {}) { + const searchParams = useSearchParams(); + const router = useRouter(); + const editing = !!initialRequest; + const adminEdit = editing && editMode === "admin"; + const requesterEdit = editing && editMode === "requester"; + // Only PENDING requests can be edited/approved. Once a request has moved on + // (provisioning/indexing/complete/failed/rejected/cancelled) the form is shown + // read-only — Save/Approve hidden, fields dimmed — so it's clear nothing can + // change from here. + const readOnly = editing && initialRequest!.status !== "pending"; + const p = initialRequest?.payload; + + // "Submit a request" vs "My requests" — surfaced as top tabs so status isn't + // buried at the bottom. Deep-linkable via ?view=requests (e.g. from the nav). + const [tab, setTab] = useState<"form" | "requests">( + searchParams?.get("view") === "requests" ? "requests" : "form" + ); + const [teamName, setTeamName] = useState(p?.team_name ?? ""); + const [channelInput, setChannelInput] = useState( + p?.channel?.channel_id ?? "" + ); + const [channel, setChannel] = useState<{ id: string; name: string } | null>( + p ? { id: p.channel.channel_id, name: p.channel.channel_name } : null + ); + const [systemPrompt, setSystemPrompt] = useState(p?.system_prompt ?? ""); + const [smeEnabled, setSmeEnabled] = useState(p?.sme?.enabled ?? false); + const [smeGroup, setSmeGroup] = useState(p?.sme?.group_name ?? ""); + const [oncallEnabled, setOncallEnabled] = useState( + p?.oncall?.enabled ?? false + ); + const [oncallSchedule, setOncallSchedule] = useState( + p?.oncall?.schedule ?? "" + ); + const [oncallHandles, setOncallHandles] = useState(p?.oncall?.handles ?? ""); + + const [docsCloud, setDocsCloud] = useState(""); + const [docsOnprem, setDocsOnprem] = useState(""); + const [extraSources, setExtraSources] = useState( + p ? prefillExtraSources(p.sources, p.channel.channel_name) : [] + ); + + const [submitting, setSubmitting] = useState(false); + const [error, setError] = useState(null); + const [submitted, setSubmitted] = useState(false); + const [savedNote, setSavedNote] = useState(null); + const [mine, setMine] = useState([]); + const [statuses, setStatuses] = useState< + Record + >({}); + + useEffect(() => { + if (editing) return; // edit mode is fully prefilled from the request + fetch("/api/onboarding/default-prompt") + .then((r) => (r.ok ? r.json() : null)) + .then((d) => d?.system_prompt && setSystemPrompt(d.system_prompt)) + .catch(() => {}); + void refreshMine(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + async function refreshMine(): Promise { + try { + const r = await fetch("/api/onboarding/mine"); + if (r.ok) { + const data = (await r.json()) as OnboardingRequestSnapshot[]; + setMine(data); + return data; + } + } catch { + /* ignore */ + } + return []; + } + + async function loadStatus(id: number) { + const r = await fetch(`/api/onboarding/${id}/status`); + if (r.ok) { + const data = (await r.json()) as OnboardingStatusResponse; + setStatuses((prev) => ({ ...prev, [id]: data })); + } + } + + // Requester withdraws a still-pending request. Soft cancel: the backend flips + // status to "cancelled" (kept for the audit trail), so just refresh the list. + async function cancelRequest(id: number) { + if (!window.confirm("Cancel this onboarding request?")) return; + const r = await fetch(`/api/onboarding/${id}/cancel`, { method: "POST" }); + if (r.ok) { + await refreshMine(); + } + } + + // On the "My requests" tab, auto-load each request's per-source scrape status + // (and re-poll every 15s) so the requester can see work happening without + // clicking anything. + useEffect(() => { + if (editing || tab !== "requests") return; + let active = true; + const load = async () => { + const list = await refreshMine(); + if (!active) return; + await Promise.all( + list + .filter((r) => (r.cc_pair_ids?.length ?? 0) > 0) + .map((r) => loadStatus(r.id)) + ); + }; + void load(); + const interval = setInterval(load, 15000); + return () => { + active = false; + clearInterval(interval); + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [tab]); + + function buildSources(): OnboardingSource[] { + const s: OnboardingSource[] = []; + if (docsCloud.trim()) + s.push({ type: "web", value: docsCloud.trim(), label: "Docs (cloud)" }); + if (docsOnprem.trim()) + s.push({ + type: "web", + value: docsOnprem.trim(), + label: "Docs (on-prem)", + }); + // The bot channel's message history is always indexed. + if (channel) + s.push({ + type: "slack", + value: channel.name, + label: `#${channel.name} history`, + }); + extraSources.forEach((x) => x.value.trim() && s.push(x)); + return s; + } + + // Assemble the request payload from form state, or null if a required field + // is missing (sets the error message). + function buildPayload(): object | null { + if (!teamName.trim()) { + setError("Enter a team name."); + return null; + } + if (!channel) { + setError("Validate the bot channel first."); + return null; + } + const sources = buildSources(); + if (sources.length === 0) { + setError("Add at least one source to index."); + return null; + } + return { + team_name: teamName.trim(), + channel: { channel_id: channel.id, channel_name: channel.name }, + // Response format is standardized to citations for every channel. + response_type: "citations", + respond_tag_only: false, + system_prompt: systemPrompt, + task_prompt: "", + sme: { enabled: smeEnabled, group_name: smeGroup }, + oncall: { + enabled: oncallEnabled, + schedule: oncallSchedule, + handles: oncallHandles, + }, + sources, + }; + } + + async function errDetail(res: Response, fallback: string): Promise { + return (await res.json().catch(() => null))?.detail || fallback; + } + + // Edit mode: PATCH the pending request's payload (admin uses the admin route, + // the requester their own). Does not approve. + async function saveEdits(): Promise { + if (!initialRequest) return false; + setError(null); + setSavedNote(null); + const payload = buildPayload(); + if (!payload) return false; + const base = adminEdit ? "/api/admin/onboarding" : "/api/onboarding"; + const res = await fetch(`${base}/${initialRequest.id}`, { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(payload), + }); + if (!res.ok) { + setError(await errDetail(res, `Couldn't save changes (${res.status}).`)); + return false; + } + return true; + } + + async function submit() { + setError(null); + setSubmitted(false); + setSavedNote(null); + const payload = buildPayload(); + if (!payload) return; + + setSubmitting(true); + try { + // Admin edit mode: save the edits, then approve + provision. + if (adminEdit && initialRequest) { + if (!(await saveEdits())) return; + const approve = await fetch( + `/api/admin/onboarding/${initialRequest.id}/approve`, + { method: "POST" } + ); + if (!approve.ok) { + setError(await errDetail(approve, `Saved, but approve failed.`)); + return; + } + router.push("/admin/onboarding"); + return; + } + + // Requester edit mode: save the edits, then back to My requests. + if (requesterEdit && initialRequest) { + if (!(await saveEdits())) return; + router.push("/onboarding?view=requests"); + return; + } + + // Requester create mode. + const res = await fetch("/api/onboarding", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(payload), + }); + if (!res.ok) { + setError( + await errDetail(res, `Couldn't submit the request (${res.status}).`) + ); + return; + } + setDocsCloud(""); + setDocsOnprem(""); + setExtraSources([]); + setSubmitted(true); + await refreshMine(); + } catch { + setError("Something went wrong — please try again."); + } finally { + setSubmitting(false); + } + } + + return ( +
+
+

+ {readOnly + ? "Onboarding request" + : adminEdit + ? "Review onboarding request" + : requesterEdit + ? "Edit onboarding request" + : "Team onboarding"} +

+

+ Onboarding Darwin to Slack Channel +

+ {readOnly ? ( +

+ Submitted by{" "} + + {initialRequest?.requester_email} + + . This request is{" "} + + {initialRequest?.status} + {" "} + — editing and approval are no longer available. +

+ ) : adminEdit ? ( +

+ Submitted by{" "} + + {initialRequest?.requester_email} + + . Fix any gaps below, then Save & approve to provision. +

+ ) : requesterEdit ? ( +

+ Update the details of your pending request. Changes are saved for + the admin to review — the request stays pending until it's + approved. +

+ ) : ( +

+ Point Darwin at your team's knowledge and channel. An admin + approves the request, then Darwin scrapes your sources and wires up + the assistant. Every field is checked live before you submit. +

+ )} +
+ + {!editing && ( +
+ {(["form", "requests"] as const).map((t) => ( + + ))} +
+ )} + + {(editing || tab === "form") && ( + <> +
+ + setChannel( + r.valid + ? { + id: String(r.resolved.channel_id ?? ""), + name: String(r.resolved.channel_name ?? ""), + } + : null + ) + } + /> + + setTeamName(e.target.value)} + placeholder="e.g. Integration Service" + className={inputClass} + /> +
+ +
+ + +

+ Paste the product root URL only. For automation-suite docs the + version is stripped and Darwin scrapes the latest 3 versions + automatically — you don't need to list each version. +

+ +
+ Additional sources +
+ {extraSources.map((s, i) => ( + 0} + canMoveDown={i < extraSources.length - 1} + onChange={(ns) => + setExtraSources((p) => p.map((x, j) => (j === i ? ns : x))) + } + onRemove={() => + setExtraSources((p) => p.filter((_, j) => j !== i)) + } + onMove={(dir) => + setExtraSources((p) => { + const j = i + dir; + if (j < 0 || j >= p.length) return p; + const c = [...p]; + [c[i], c[j]] = [c[j], c[i]]; + return c; + }) + } + /> + ))} + +
+ +
+