From ad900a854c8d44e5d531c8c24131304f83a07f5d Mon Sep 17 00:00:00 2001 From: rajiv chodisetti Date: Fri, 17 Jul 2026 14:49:46 +0530 Subject: [PATCH 01/40] feat(onboarding): self-serve team onboarding form + admin approval + auto-provision Anyone can submit an onboarding request; admins approve. On approval we auto-provision connectors (web docs all-versions, Confluence, GitHub, Slack), an assistant (Orchestrator prompt by default, requester-editable), a Slack bot config (optional SME review / oncall / Jira), and high-priority indexing, then expose per-source scrape status the requester can monitor. - backend: OnboardingRequest model + status enum (stored as String to avoid the native_enum NAME/value gotcha), migration c40f5a073fc2, db/onboarding.py CRUD, onboarding/validation.py (inline Slack channel/group, Confluence space, docs root validation) + onboarding/provision.py orchestrator, and the /onboarding + /admin/onboarding API. - security: require an authenticated user on submit/validate/status (reject anonymous / API-key service callers) and a Confluence host allowlist so the stored Confluence token is only ever sent to a host an existing Confluence connector already uses (SSRF / credential-leak guard). - web: /onboarding request form (inline field validation, cloud+on-prem docs roots, editable prompt, SME/oncall/Jira toggles, my-requests + scrape status) and /admin/onboarding approval queue. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../c40f5a073fc2_onboarding_request.py | 75 +++ backend/danswer/db/models.py | 68 ++ backend/danswer/db/onboarding.py | 103 ++++ backend/danswer/main.py | 6 + backend/danswer/onboarding/__init__.py | 0 backend/danswer/onboarding/provision.py | 372 +++++++++++ backend/danswer/onboarding/validation.py | 225 +++++++ .../server/features/onboarding/__init__.py | 0 .../danswer/server/features/onboarding/api.py | 244 ++++++++ .../server/features/onboarding/models.py | 107 ++++ .../tests/unit/danswer/onboarding/__init__.py | 0 .../unit/danswer/onboarding/test_provision.py | 115 ++++ .../danswer/onboarding/test_validation.py | 31 + .../onboarding/OnboardingRequestsTable.tsx | 169 +++++ web/src/app/admin/onboarding/page.tsx | 19 + web/src/app/onboarding/OnboardingForm.tsx | 579 ++++++++++++++++++ web/src/app/onboarding/page.tsx | 5 + web/src/lib/onboarding/interfaces.ts | 93 +++ 18 files changed, 2211 insertions(+) create mode 100644 backend/alembic/versions/c40f5a073fc2_onboarding_request.py create mode 100644 backend/danswer/db/onboarding.py create mode 100644 backend/danswer/onboarding/__init__.py create mode 100644 backend/danswer/onboarding/provision.py create mode 100644 backend/danswer/onboarding/validation.py create mode 100644 backend/danswer/server/features/onboarding/__init__.py create mode 100644 backend/danswer/server/features/onboarding/api.py create mode 100644 backend/danswer/server/features/onboarding/models.py create mode 100644 backend/tests/unit/danswer/onboarding/__init__.py create mode 100644 backend/tests/unit/danswer/onboarding/test_provision.py create mode 100644 backend/tests/unit/danswer/onboarding/test_validation.py create mode 100644 web/src/app/admin/onboarding/OnboardingRequestsTable.tsx create mode 100644 web/src/app/admin/onboarding/page.tsx create mode 100644 web/src/app/onboarding/OnboardingForm.tsx create mode 100644 web/src/app/onboarding/page.tsx create mode 100644 web/src/lib/onboarding/interfaces.ts diff --git a/backend/alembic/versions/c40f5a073fc2_onboarding_request.py b/backend/alembic/versions/c40f5a073fc2_onboarding_request.py new file mode 100644 index 00000000000..c67cf111b27 --- /dev/null +++ b/backend/alembic/versions/c40f5a073fc2_onboarding_request.py @@ -0,0 +1,75 @@ +"""onboarding_request + +Revision ID: c40f5a073fc2 +Revises: 31f30b318163 +Create Date: 2026-07-17 00:00:00.000000 + +Adds the onboarding_request table backing the self-serve team-onboarding flow +(form -> admin approval -> auto-provision + scrape). All columns additive/new +table, so no impact on existing data. +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "c40f5a073fc2" +down_revision = "31f30b318163" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "onboarding_request", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "requester_id", + postgresql.UUID(as_uuid=True), + sa.ForeignKey("user.id"), + nullable=True, + ), + sa.Column("requester_email", sa.String(), nullable=False), + sa.Column("status", sa.String(), nullable=False, server_default="pending"), + sa.Column("payload", postgresql.JSONB(), nullable=False), + sa.Column( + "approver_id", + postgresql.UUID(as_uuid=True), + sa.ForeignKey("user.id"), + nullable=True, + ), + sa.Column("decision_reason", sa.Text(), nullable=True), + sa.Column("error_msg", sa.Text(), nullable=True), + sa.Column( + "persona_id", sa.Integer(), sa.ForeignKey("persona.id"), nullable=True + ), + sa.Column( + "document_set_id", + sa.Integer(), + sa.ForeignKey("document_set.id"), + nullable=True, + ), + sa.Column( + "slack_bot_config_id", + sa.Integer(), + sa.ForeignKey("slack_bot_config.id"), + nullable=True, + ), + sa.Column("cc_pair_ids", postgresql.JSONB(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + ) + + +def downgrade() -> None: + op.drop_table("onboarding_request") diff --git a/backend/danswer/db/models.py b/backend/danswer/db/models.py index ff01513bf14..28e4ef476f3 100644 --- a/backend/danswer/db/models.py +++ b/backend/danswer/db/models.py @@ -1716,3 +1716,71 @@ class ChatReferral(Base): created_at: Mapped[datetime.datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() ) + + +class OnboardingStatus(str, PyEnum): + """Lifecycle of a team's self-serve Darwin onboarding request.""" + + PENDING = "pending" # submitted, awaiting admin approval + REJECTED = "rejected" # admin declined + PROVISIONING = "provisioning" # approved; creating connectors/persona/config + INDEXING = "indexing" # resources created; sources scraping + COMPLETE = "complete" # all sources indexed successfully + FAILED = "failed" # provisioning or indexing failed + + +class OnboardingRequest(Base): + """A self-serve request to onboard a team/channel onto Darwin. Anyone may + submit; only an admin may approve. On approval the provisioning orchestrator + (onboarding/provision.py) creates the connectors, document set, assistant, and + Slack bot config, then records their ids here so the requester can monitor the + per-source scrape status.""" + + __tablename__ = "onboarding_request" + + id: Mapped[int] = mapped_column(primary_key=True) + # Who submitted (denormalize email so it survives user deletion / is display-ready). + requester_id: Mapped[UUID | None] = mapped_column( + ForeignKey("user.id"), nullable=True + ) + requester_email: Mapped[str] = mapped_column(String, nullable=False) + # Stored as the enum VALUE string (e.g. "pending"), NOT via SA Enum() — the + # repo's Enum(native_enum=False) stores the NAME (uppercase), which is a known + # footgun. Plain String of `.value` keeps it consistent with the migration. + status: Mapped[str] = mapped_column( + String, nullable=False, default=OnboardingStatus.PENDING.value + ) + # The full validated form (channel, ordered sources, prompt, SME/oncall/jira + # options, docs cloud/onprem roots, etc.) — JSONB for flexibility. + payload: Mapped[dict] = mapped_column(postgresql.JSONB(), nullable=False) + # Admin who approved/rejected + optional reason. + approver_id: Mapped[UUID | None] = mapped_column( + ForeignKey("user.id"), nullable=True + ) + decision_reason: Mapped[str | None] = mapped_column(Text, nullable=True) + # Free-text error surfaced when status == FAILED. + error_msg: Mapped[str | None] = mapped_column(Text, nullable=True) + # Provisioned resource ids (populated on approval; drive the status monitor). + persona_id: Mapped[int | None] = mapped_column( + ForeignKey("persona.id"), nullable=True + ) + document_set_id: Mapped[int | None] = mapped_column( + ForeignKey("document_set.id"), nullable=True + ) + slack_bot_config_id: Mapped[int | None] = mapped_column( + ForeignKey("slack_bot_config.id"), nullable=True + ) + # cc_pair ids created for this onboarding (list[int]); the monitor reads their + # indexing status. JSONB list rather than a join table — always used together. + cc_pair_ids: Mapped[list[int] | None] = mapped_column( + postgresql.JSONB(), nullable=True + ) + created_at: Mapped[datetime.datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime.datetime] = mapped_column( + DateTime(timezone=True), + nullable=False, + server_default=func.now(), + onupdate=func.now(), + ) diff --git a/backend/danswer/db/onboarding.py b/backend/danswer/db/onboarding.py new file mode 100644 index 00000000000..e60b47ee3d6 --- /dev/null +++ b/backend/danswer/db/onboarding.py @@ -0,0 +1,103 @@ +"""DB helpers for the self-serve Darwin onboarding flow (see OnboardingRequest). + +Status is stored as the enum VALUE string; always write/compare with +OnboardingStatus(...).value to stay consistent with the column + migration.""" +from uuid import UUID + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from danswer.db.models import OnboardingRequest +from danswer.db.models import OnboardingStatus + + +def create_onboarding_request( + db_session: Session, + requester_id: UUID | None, + requester_email: str, + payload: dict, +) -> OnboardingRequest: + request = OnboardingRequest( + requester_id=requester_id, + requester_email=requester_email, + payload=payload, + status=OnboardingStatus.PENDING.value, + ) + db_session.add(request) + db_session.commit() + return request + + +def get_onboarding_request( + db_session: Session, request_id: int +) -> OnboardingRequest | None: + return db_session.get(OnboardingRequest, request_id) + + +def list_onboarding_requests( + db_session: Session, status: OnboardingStatus | None = None +) -> list[OnboardingRequest]: + """All requests (admin view), newest first; optionally filtered by status.""" + stmt = select(OnboardingRequest).order_by(OnboardingRequest.created_at.desc()) + if status is not None: + stmt = stmt.where(OnboardingRequest.status == status.value) + return list(db_session.execute(stmt).scalars().all()) + + +def list_onboarding_requests_for_user( + db_session: Session, requester_id: UUID +) -> list[OnboardingRequest]: + """A requester's own submissions, newest first.""" + stmt = ( + select(OnboardingRequest) + .where(OnboardingRequest.requester_id == requester_id) + .order_by(OnboardingRequest.created_at.desc()) + ) + return list(db_session.execute(stmt).scalars().all()) + + +def update_onboarding_status( + db_session: Session, + request: OnboardingRequest, + status: OnboardingStatus, + *, + approver_id: UUID | None = None, + decision_reason: str | None = None, + error_msg: str | None = None, + commit: bool = True, +) -> OnboardingRequest: + request.status = status.value + if approver_id is not None: + request.approver_id = approver_id + if decision_reason is not None: + request.decision_reason = decision_reason + # error_msg is cleared on a non-failed transition, set on failure. + request.error_msg = error_msg if status == OnboardingStatus.FAILED else None + if commit: + db_session.commit() + return request + + +def set_provisioned_ids( + db_session: Session, + request: OnboardingRequest, + *, + persona_id: int | None = None, + document_set_id: int | None = None, + slack_bot_config_id: int | None = None, + cc_pair_ids: list[int] | None = None, + commit: bool = True, +) -> OnboardingRequest: + """Record the resources provisioning created, so the status monitor can map + the request to its cc_pairs / assistant.""" + if persona_id is not None: + request.persona_id = persona_id + if document_set_id is not None: + request.document_set_id = document_set_id + if slack_bot_config_id is not None: + request.slack_bot_config_id = slack_bot_config_id + if cc_pair_ids is not None: + request.cc_pair_ids = cc_pair_ids + if commit: + db_session.commit() + return request diff --git a/backend/danswer/main.py b/backend/danswer/main.py index 879d1edb0dd..d0ab43109db 100644 --- a/backend/danswer/main.py +++ b/backend/danswer/main.py @@ -63,6 +63,10 @@ from danswer.server.documents.document import router as document_router from danswer.server.features.document_set.api import router as document_set_router from danswer.server.features.folder.api import router as folder_router +from danswer.server.features.onboarding.api import ( + admin_router as admin_onboarding_router, +) +from danswer.server.features.onboarding.api import basic_router as onboarding_router from danswer.server.features.persona.api import admin_router as admin_persona_router from danswer.server.features.persona.api import basic_router as persona_router from danswer.server.features.prompt.api import basic_router as prompt_router @@ -280,6 +284,8 @@ def get_application() -> FastAPI: ) include_router_with_global_prefix_prepended(application, persona_router) include_router_with_global_prefix_prepended(application, admin_persona_router) + include_router_with_global_prefix_prepended(application, onboarding_router) + include_router_with_global_prefix_prepended(application, admin_onboarding_router) include_router_with_global_prefix_prepended(application, prompt_router) include_router_with_global_prefix_prepended(application, tool_router) include_router_with_global_prefix_prepended(application, admin_tool_router) diff --git a/backend/danswer/onboarding/__init__.py b/backend/danswer/onboarding/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/backend/danswer/onboarding/provision.py b/backend/danswer/onboarding/provision.py new file mode 100644 index 00000000000..f34e14f9589 --- /dev/null +++ b/backend/danswer/onboarding/provision.py @@ -0,0 +1,372 @@ +"""Provisioning orchestrator for approved onboarding requests. + +On admin approval this turns a validated onboarding payload into real Darwin +resources, in order: + per source -> Connector (+ credential) -> connector_credential_pair + -> DocumentSet (over all cc_pairs) + -> Prompt (Orchestrator's as the default template, overridden by the requester) + -> Persona (the team's assistant, scoped to the document set) + -> slack_bot_config (channel -> persona, with SME / oncall / Jira options and + the requested source order as prioritized_sources) + -> one high-priority IndexAttempt per cc_pair (so the new sources scrape first) +and records the created ids on the OnboardingRequest for status monitoring. + +Payload contract (assembled + validated by the form): + { + "team_name": str, + "channel": {"channel_id": str, "channel_name": str}, + "response_type": "citations" | "quotes", + "respond_tag_only": bool, + "system_prompt": str, "task_prompt": str, # requester-edited + "sme": {"enabled": bool, "group_name": str}, + "oncall": {"enabled": bool, "schedule": str}, # opsgenie_schedule + "jira": {"enabled": bool, "project_key": str, "issue_type": str, "component": str}, + "sources": [ {"type": "web"|"confluence"|"github"|"slack", + "value": , "label": str}, ... ] # priority order + } +""" +from urllib.parse import urlparse + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from danswer.configs.constants import DocumentSource +from danswer.connectors.models import InputType +from danswer.db.connector import create_connector +from danswer.db.connector_credential_pair import add_credential_to_connector +from danswer.db.document_set import insert_document_set +from danswer.db.embedding_model import get_current_db_embedding_model +from danswer.db.index_attempt import create_index_attempt +from danswer.db.models import ChannelConfig +from danswer.db.models import Connector +from danswer.db.models import Credential +from danswer.db.models import OnboardingRequest +from danswer.db.models import OnboardingStatus +from danswer.db.models import Prompt +from danswer.db.models import RecencyBiasSetting +from danswer.db.models import SlackBotResponseType +from danswer.db.models import User +from danswer.db.onboarding import set_provisioned_ids +from danswer.db.onboarding import update_onboarding_status +from danswer.db.persona import get_persona_by_name +from danswer.db.persona import upsert_persona +from danswer.db.persona import upsert_prompt +from danswer.db.slack_bot_config import insert_slack_bot_config +from danswer.server.documents.models import ConnectorBase +from danswer.server.features.document_set.models import DocumentSetCreationRequest +from danswer.utils.logger import setup_logger + +logger = setup_logger() + +# Onboarded sources scrape ahead of routine re-indexing (IndexAttempt priority 0-100). +ONBOARDING_INDEXING_PRIORITY = 80 +DEFAULT_REFRESH_FREQ = 86400 # daily; picks up new docs versions / channel messages +PUBLIC_CREDENTIAL_ID = ( + 0 # the empty public credential (create_initial_public_credential) +) +_DEFAULT_TEMPLATE_PERSONA = "Orchestrator" + + +def _find_credential_id(db_session: Session, required_key: str) -> int | None: + """Reuse an existing connector's shared credential for auth'd sources + (Confluence/GitHub/Slack) — the requester never supplies tokens.""" + for cred in db_session.execute(select(Credential)).scalars(): + if (cred.credential_json or {}).get(required_key): + return cred.id + return None + + +def _slack_workspace(db_session: Session) -> str | None: + for connector in ( + db_session.execute( + select(Connector).where(Connector.source == DocumentSource.SLACK) + ) + .scalars() + .all() + ): + ws = (connector.connector_specific_config or {}).get("workspace") + if ws: + return ws + return None + + +def _parse_github_repo(url: str) -> tuple[str, str]: + parts = urlparse(url).path.strip("/").split("/") + if len(parts) < 2: + raise ValueError(f"Not a github repo URL: {url}") + return parts[0], parts[1] + + +def _build_connector_base(source: dict, db_session: Session) -> ConnectorBase: + """One onboarding source -> a ConnectorBase (source-specific config).""" + stype = source["type"] + value = source["value"] + label = source.get("label") or value + + if stype == "web": + is_uipath_docs = urlparse(value).netloc == "docs.uipath.com" + config: dict = {"base_url": value, "web_connector_type": "recursive"} + if is_uipath_docs: + # Take the root URL and let the connector crawl all product versions. + config["uipath_latest_versions"] = True + config["max_versions"] = 2 + return ConnectorBase( + name=f"[onboarding] {label}", + source=DocumentSource.WEB, + input_type=InputType.POLL, + connector_specific_config=config, + refresh_freq=DEFAULT_REFRESH_FREQ, + prune_freq=None, + disabled=False, + ) + if stype == "confluence": + return ConnectorBase( + name=f"[onboarding] {label}", + source=DocumentSource.CONFLUENCE, + input_type=InputType.POLL, + connector_specific_config={"wiki_page_url": value}, + refresh_freq=DEFAULT_REFRESH_FREQ, + prune_freq=None, + disabled=False, + ) + if stype == "github": + owner, repo = _parse_github_repo(value) + return ConnectorBase( + name=f"[onboarding] {label}", + source=DocumentSource.GITHUB, + input_type=InputType.POLL, + connector_specific_config={ + "repo_owner": owner, + "repo_name": repo, + "include_prs": True, + "include_issues": True, + }, + refresh_freq=DEFAULT_REFRESH_FREQ, + prune_freq=None, + disabled=False, + ) + if stype == "slack": + workspace = _slack_workspace(db_session) + if not workspace: + raise ValueError("No existing Slack connector to copy the workspace from") + return ConnectorBase( + name=f"[onboarding] {label}", + source=DocumentSource.SLACK, + input_type=InputType.POLL, + connector_specific_config={ + "workspace": workspace, + "channels": [value.lstrip("#")], + "channel_regex_enabled": False, + }, + refresh_freq=DEFAULT_REFRESH_FREQ, + prune_freq=None, + disabled=False, + ) + raise ValueError(f"Unsupported source type: {stype}") + + +# Which decrypted-credential key identifies a reusable credential per source type. +_CREDENTIAL_KEY_BY_SOURCE = { + "confluence": "confluence_access_token", + "github": "github_access_token", + "slack": "slack_bot_token", +} + + +def _credential_id_for_source(source_type: str, db_session: Session) -> int: + if source_type == "web": + return PUBLIC_CREDENTIAL_ID + key = _CREDENTIAL_KEY_BY_SOURCE.get(source_type) + if key is None: + return PUBLIC_CREDENTIAL_ID + cred_id = _find_credential_id(db_session, key) + if cred_id is None: + raise ValueError( + f"No existing {source_type} credential to reuse (missing '{key}')" + ) + return cred_id + + +def _source_type_value(source_type: str) -> str: + """Onboarding source type -> DocumentSource value (for prioritized_sources).""" + return { + "web": DocumentSource.WEB.value, + "confluence": DocumentSource.CONFLUENCE.value, + "github": DocumentSource.GITHUB.value, + "slack": DocumentSource.SLACK.value, + }.get(source_type, source_type) + + +def _build_prompt(payload: dict, admin_user: User, db_session: Session) -> Prompt: + """Create the team's prompt, defaulting to the Orchestrator persona's prompt + and overriding with whatever the requester edited in the form.""" + template = get_persona_by_name(_DEFAULT_TEMPLATE_PERSONA, admin_user, db_session) + default_system = "" + default_task = "" + if template and template.prompts: + default_system = template.prompts[0].system_prompt + default_task = template.prompts[0].task_prompt + team = payload["team_name"] + return upsert_prompt( + user=admin_user, + name=f"[onboarding] {team} prompt", + description=f"Prompt for the {team} assistant (from onboarding)", + system_prompt=payload.get("system_prompt") or default_system, + task_prompt=payload.get("task_prompt") or default_task, + include_citations=True, + datetime_aware=True, + personas=None, + db_session=db_session, + default_prompt=False, + ) + + +def _build_channel_config( + payload: dict, prioritized_sources: list[str] +) -> ChannelConfig: + channel_name = payload["channel"]["channel_name"] + config: ChannelConfig = { + "channel_names": [channel_name], + "respond_tag_only": bool(payload.get("respond_tag_only", False)), + "prioritized_sources": prioritized_sources, + } + sme = payload.get("sme") or {} + if sme.get("enabled"): + config["enable_sme_validation"] = True + config["sme_group_name"] = sme.get("group_name", "") + oncall = payload.get("oncall") or {} + if oncall.get("enabled") and oncall.get("schedule"): + config["opsgenie_schedule"] = oncall["schedule"] + jira = payload.get("jira") or {} + if jira.get("enabled"): + config["jira_config"] = { + "enable_jira_integration": True, + "project_key": jira.get("project_key", ""), + "issue_type": jira.get("issue_type", ""), + "component": jira.get("component", ""), + } + return config + + +def provision_onboarding( + request: OnboardingRequest, + admin_user: User, + db_session: Session, +) -> OnboardingRequest: + """Create all resources for an approved request. On any failure the request + is marked FAILED with the error and the exception is re-raised.""" + payload = request.payload + update_onboarding_status( + db_session, request, OnboardingStatus.PROVISIONING, approver_id=admin_user.id + ) + try: + embedding_model = get_current_db_embedding_model(db_session) + team = payload["team_name"] + + # 1) connectors + cc_pairs (track connector/credential ids for indexing). + cc_pair_ids: list[int] = [] + index_targets: list[tuple[int, int]] = [] # (connector_id, credential_id) + prioritized_sources: list[str] = [] + for source in payload["sources"]: + connector = create_connector( + _build_connector_base(source, db_session), db_session + ) + connector_id = int(connector.id) + credential_id = _credential_id_for_source(source["type"], db_session) + ccp = add_credential_to_connector( + connector_id=connector_id, + credential_id=credential_id, + cc_pair_name=f"[onboarding] {team}: {source.get('label') or source['value']}", + is_public=True, + user=admin_user, + db_session=db_session, + ) + if ccp.data is None: + raise ValueError(f"Failed to create cc_pair for {source['value']}") + cc_pair_ids.append(ccp.data) + index_targets.append((connector_id, credential_id)) + st = _source_type_value(source["type"]) + if st not in prioritized_sources: + prioritized_sources.append(st) + + # 2) document set over all cc_pairs. + doc_set, _ = insert_document_set( + DocumentSetCreationRequest( + name=f"[onboarding] {team}", + description=f"Sources onboarded for {team}", + cc_pair_ids=cc_pair_ids, + is_public=True, + ), + admin_user.id, + db_session, + ) + + # 3) prompt (Orchestrator default + requester edits) + persona. + prompt = _build_prompt(payload, admin_user, db_session) + persona = upsert_persona( + user=admin_user, + name=team, + description=f"Assistant for {team} (self-serve onboarding)", + num_chunks=10, + llm_relevance_filter=False, + llm_filter_extraction=False, + recency_bias=RecencyBiasSetting.BASE_DECAY, + llm_model_provider_override=None, + llm_model_version_override=None, + starter_messages=None, + is_public=True, + db_session=db_session, + prompt_ids=[prompt.id], + document_set_ids=[doc_set.id], + ) + + # 4) slack bot config (channel -> persona) with the options + priority order. + channel_config = _build_channel_config(payload, prioritized_sources) + response_type = ( + SlackBotResponseType.QUOTES + if payload.get("response_type") == "quotes" + else SlackBotResponseType.CITATIONS + ) + slack_config = insert_slack_bot_config( + persona_id=persona.id, + channel_config=channel_config, + response_type=response_type, + db_session=db_session, + ) + + set_provisioned_ids( + db_session, + request, + persona_id=persona.id, + document_set_id=doc_set.id, + slack_bot_config_id=slack_config.id, + cc_pair_ids=cc_pair_ids, + ) + + # 5) kick off high-priority indexing for each new source. + for connector_id, credential_id in index_targets: + create_index_attempt( + connector_id=connector_id, + credential_id=credential_id, + embedding_model_id=embedding_model.id, + db_session=db_session, + from_beginning=True, + indexing_priority=ONBOARDING_INDEXING_PRIORITY, + ) + + update_onboarding_status(db_session, request, OnboardingStatus.INDEXING) + logger.info( + "onboarding %s provisioned: persona=%s doc_set=%s config=%s cc_pairs=%s", + request.id, + persona.id, + doc_set.id, + slack_config.id, + cc_pair_ids, + ) + return request + except Exception as e: + logger.exception("onboarding %s provisioning failed", request.id) + update_onboarding_status( + db_session, request, OnboardingStatus.FAILED, error_msg=str(e) + ) + raise diff --git a/backend/danswer/onboarding/validation.py b/backend/danswer/onboarding/validation.py new file mode 100644 index 00000000000..17762504ec4 --- /dev/null +++ b/backend/danswer/onboarding/validation.py @@ -0,0 +1,225 @@ +"""Inline validation for the self-serve onboarding form. + +Every source/handle a requester enters is validated against the live system +before submission: Slack channels + user-groups via the bot's Slack client, +Confluence spaces via an existing Confluence connector's credentials, and +docs.uipath.com roots via the web-connector's version logic. Each validator is +resilient — it distinguishes "invalid" from "couldn't check right now".""" +import re +from urllib.parse import urlparse + +from pydantic import BaseModel +from slack_sdk import WebClient +from slack_sdk.errors import SlackApiError +from sqlalchemy import select +from sqlalchemy.orm import Session + +from danswer.configs.constants import DocumentSource +from danswer.connectors.confluence.connector import extract_confluence_keys_from_url +from danswer.connectors.web.connector import _uipath_product_prefix +from danswer.danswerbot.slack.tokens import fetch_tokens +from danswer.danswerbot.slack.utils import fetch_groupids_from_names +from danswer.db.models import Connector +from danswer.db.models import Credential +from danswer.utils.logger import setup_logger + +logger = setup_logger() + +# Bounded scan for name->channel lookup (Slack has no name lookup API). Prefer a +# #mention (carries the id) — that path is a single, exact call. +_MAX_CHANNEL_PAGES = 12 +_MENTION_RE = re.compile(r"<#(C[A-Z0-9]+)(?:\|[^>]*)?>") +_CHANNEL_ID_RE = re.compile(r"^C[A-Z0-9]{6,}$") + + +class ValidationResult(BaseModel): + valid: bool + message: str + # Resolved canonical values (e.g. channel id/name, wiki base) when valid. + resolved: dict = {} + + +def _bot_client() -> WebClient: + return WebClient(token=fetch_tokens().bot_token) + + +def validate_slack_channel(value: str) -> ValidationResult: + """Accepts a #mention, a channel id, or a bare name. Resolves to the real + channel and confirms the bot can see it.""" + raw = (value or "").strip() + if not raw: + return ValidationResult(valid=False, message="Enter a channel") + + mention = _MENTION_RE.search(raw) + channel_id = ( + mention.group(1) if mention else (raw if _CHANNEL_ID_RE.match(raw) else None) + ) + try: + client = _bot_client() + except Exception as e: + logger.warning("slack client unavailable for validation: %s", e) + return ValidationResult(valid=False, message="Couldn't reach Slack to verify") + + if channel_id: + try: + ch = client.conversations_info(channel=channel_id)["channel"] + return ValidationResult( + valid=True, + message=f"#{ch['name']}", + resolved={"channel_id": ch["id"], "channel_name": ch["name"]}, + ) + except SlackApiError as e: + return ValidationResult( + valid=False, + message=f"Channel not found or bot lacks access ({e.response.get('error')})", + ) + + name = raw.lstrip("#").lower() + cursor: str | None = None + try: + for _ in range(_MAX_CHANNEL_PAGES): + resp = client.conversations_list( + types="public_channel,private_channel", + limit=1000, + cursor=cursor, + exclude_archived=True, + ) + for ch in resp.get("channels", []): + if ch.get("name", "").lower() == name: + return ValidationResult( + valid=True, + message=f"#{ch['name']}", + resolved={"channel_id": ch["id"], "channel_name": ch["name"]}, + ) + meta = resp.get("response_metadata") + cursor = meta.get("next_cursor") if isinstance(meta, dict) else None + if not cursor: + break + except SlackApiError as e: + return ValidationResult( + valid=False, message=f"Couldn't verify channel ({e.response.get('error')})" + ) + return ValidationResult( + valid=False, + message="Channel not found — paste it as a #mention to be sure", + ) + + +def validate_slack_group(name: str) -> ValidationResult: + """A Slack user-group / team name (used for SME groups + oncall).""" + raw = (name or "").strip().lstrip("@") + if not raw: + return ValidationResult(valid=False, message="Enter a group name") + try: + client = _bot_client() + group_ids, failed = fetch_groupids_from_names([raw], client) + except Exception as e: + logger.warning("slack group validation failed: %s", e) + return ValidationResult(valid=False, message="Couldn't reach Slack to verify") + if group_ids: + return ValidationResult( + valid=True, message=f"@{raw}", resolved={"group_id": group_ids[0]} + ) + return ValidationResult(valid=False, message=f"No Slack user group named '{raw}'") + + +def _first_confluence_credential(db_session: Session) -> dict | None: + """Reuse an existing Confluence connector's creds for inline space checks.""" + for cred in db_session.execute(select(Credential)).scalars(): + cj = cred.credential_json or {} + if cj.get("confluence_access_token") and cj.get("confluence_username"): + return cj + return None + + +def _allowed_confluence_hosts(db_session: Session) -> set[str]: + """Hosts of existing Confluence connectors — the ONLY hosts we'll send our + stored Confluence token to (SSRF / credential-leak guard: never let a + user-supplied URL point the authenticated client at an arbitrary host).""" + hosts: set[str] = set() + for connector in ( + db_session.execute( + select(Connector).where(Connector.source == DocumentSource.CONFLUENCE) + ) + .scalars() + .all() + ): + wiki_url = (connector.connector_specific_config or {}).get("wiki_page_url") + netloc = urlparse(wiki_url).netloc.lower() if wiki_url else "" + if netloc: + hosts.add(netloc) + return hosts + + +def validate_confluence_url(url: str, db_session: Session) -> ValidationResult: + """Parse the wiki URL and confirm the space exists, using an existing + Confluence connector's credentials — but ONLY if the URL's host matches an + existing Confluence connector (never send creds to an arbitrary host).""" + raw = (url or "").strip() + if not raw: + return ValidationResult(valid=False, message="Enter a Confluence URL") + try: + wiki_base, space, _page_id, is_cloud = extract_confluence_keys_from_url(raw) + except ValueError: + return ValidationResult(valid=False, message="Not a valid Confluence wiki URL") + + # SSRF / credential-leak guard: only proceed against a known Confluence host. + host = urlparse(wiki_base).netloc.lower() + allowed_hosts = _allowed_confluence_hosts(db_session) + if allowed_hosts and host not in allowed_hosts: + return ValidationResult( + valid=False, + message=f"Confluence host not allowed — must be one of {sorted(allowed_hosts)}", + ) + + creds = _first_confluence_credential(db_session) + # No creds, or no known Confluence host to vet against -> parse-only (never + # send the token to an unvetted host). + if creds is None or not allowed_hosts: + return ValidationResult( + valid=True, + message=f"Space '{space}' (existence unverified)", + resolved={"wiki_base": wiki_base, "space": space, "is_cloud": is_cloud}, + ) + try: + from atlassian import Confluence # type: ignore[import-untyped] + + client = Confluence( + url=wiki_base, + username=creds["confluence_username"], + password=creds["confluence_access_token"], + cloud=is_cloud, + ) + client.get_space(space) + except Exception as e: + logger.info("confluence space validation failed for %s: %s", space, e) + return ValidationResult( + valid=False, message=f"Space '{space}' not found or inaccessible" + ) + return ValidationResult( + valid=True, + message=f"Space '{space}'", + resolved={"wiki_base": wiki_base, "space": space, "is_cloud": is_cloud}, + ) + + +def validate_docs_url(url: str) -> ValidationResult: + """A docs.uipath.com root URL. We normalize to the product root (version / + 'latest' segment stripped) since the web connector auto-crawls all versions.""" + raw = (url or "").strip() + if not raw: + return ValidationResult(valid=False, message="Enter a docs URL") + if not re.match(r"^https?://docs\.uipath\.com/", raw): + return ValidationResult(valid=False, message="Must be a docs.uipath.com URL") + from urllib.parse import urlparse + + parsed = urlparse(raw) + product_prefix = _uipath_product_prefix(parsed.path) + if not product_prefix.strip("/"): + return ValidationResult(valid=False, message="URL must include a product path") + root = f"{parsed.scheme}://{parsed.netloc}{product_prefix}" + return ValidationResult( + valid=True, + message=f"Will crawl all versions under {root}", + resolved={"root_url": root}, + ) diff --git a/backend/danswer/server/features/onboarding/__init__.py b/backend/danswer/server/features/onboarding/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/backend/danswer/server/features/onboarding/api.py b/backend/danswer/server/features/onboarding/api.py new file mode 100644 index 00000000000..3c69d1c2a10 --- /dev/null +++ b/backend/danswer/server/features/onboarding/api.py @@ -0,0 +1,244 @@ +"""Self-serve onboarding API. + +- basic_router (/onboarding): any authenticated user may submit a request, watch + their own requests, validate form fields inline, and monitor scrape status. +- admin_router (/admin/onboarding): admins list/approve/reject; approval triggers + provisioning (connectors + assistant + Slack config + high-priority indexing). +""" +from fastapi import APIRouter +from fastapi import Depends +from fastapi import HTTPException +from sqlalchemy import desc +from sqlalchemy import select +from sqlalchemy.orm import Session + +from danswer.auth.users import current_admin_user +from danswer.auth.users import current_user +from danswer.configs.app_configs import DISABLE_AUTH +from danswer.db.connector_credential_pair import get_connector_credential_pair_from_id +from danswer.db.engine import get_session +from danswer.db.models import IndexAttempt +from danswer.db.models import IndexingStatus +from danswer.db.models import OnboardingRequest +from danswer.db.models import OnboardingStatus +from danswer.db.models import User +from danswer.db.models import UserRole +from danswer.db.onboarding import create_onboarding_request +from danswer.db.onboarding import get_onboarding_request +from danswer.db.onboarding import list_onboarding_requests +from danswer.db.onboarding import list_onboarding_requests_for_user +from danswer.db.onboarding import update_onboarding_status +from danswer.onboarding.provision import provision_onboarding +from danswer.onboarding.validation import validate_confluence_url +from danswer.onboarding.validation import validate_docs_url +from danswer.onboarding.validation import validate_slack_channel +from danswer.onboarding.validation import validate_slack_group +from danswer.onboarding.validation import ValidationResult +from danswer.server.features.onboarding.models import DecisionRequest +from danswer.server.features.onboarding.models import OnboardingRequestSnapshot +from danswer.server.features.onboarding.models import OnboardingStatusResponse +from danswer.server.features.onboarding.models import OnboardingSubmitRequest +from danswer.server.features.onboarding.models import SourceStatus +from danswer.server.features.onboarding.models import ValidateRequest + +basic_router = APIRouter(prefix="/onboarding") +admin_router = APIRouter(prefix="/admin/onboarding") + + +def _require_user(user: User | None) -> None: + """Onboarding is a human workflow — reject anonymous / API-key (service) + callers. `current_user` returns None for API-key callers and when auth is + globally disabled; allow None only in the latter (dev) case.""" + if user is None and not DISABLE_AUTH: + raise HTTPException(status_code=401, detail="Authentication required.") + + +# --- submit + validate + monitor (any authed user) -------------------------- + + +@basic_router.post("") +def submit_onboarding( + request: OnboardingSubmitRequest, + user: User | None = Depends(current_user), + db_session: Session = Depends(get_session), +) -> OnboardingRequestSnapshot: + _require_user(user) + if not request.sources: + raise HTTPException(status_code=400, detail="At least one source is required.") + created = create_onboarding_request( + db_session=db_session, + requester_id=user.id if user else None, + requester_email=user.email if user else "system@darwin", + payload=request.to_payload(), + ) + return OnboardingRequestSnapshot.from_model(created) + + +@basic_router.post("/validate") +def validate_field( + request: ValidateRequest, + user: User | None = Depends(current_user), + db_session: Session = Depends(get_session), +) -> ValidationResult: + """Inline validation for a single form field.""" + _require_user(user) + if request.kind == "slack_channel": + return validate_slack_channel(request.value) + if request.kind == "slack_group": + return validate_slack_group(request.value) + if request.kind == "confluence": + return validate_confluence_url(request.value, db_session) + return validate_docs_url(request.value) + + +@basic_router.get("/default-prompt") +def default_prompt( + user: User | None = Depends(current_user), + db_session: Session = Depends(get_session), +) -> dict: + """The template prompt the form prefills (the Orchestrator assistant's prompt); + the requester can edit it before submitting.""" + from danswer.db.persona import get_persona_by_name + + template = get_persona_by_name("Orchestrator", user, db_session) + if template and template.prompts: + p = template.prompts[0] + return {"system_prompt": p.system_prompt, "task_prompt": p.task_prompt} + return {"system_prompt": "", "task_prompt": ""} + + +@basic_router.get("/mine") +def my_onboarding_requests( + user: User | None = Depends(current_user), + db_session: Session = Depends(get_session), +) -> list[OnboardingRequestSnapshot]: + if user is None: + return [] + return [ + OnboardingRequestSnapshot.from_model(r) + for r in list_onboarding_requests_for_user(db_session, user.id) + ] + + +def _source_statuses( + request: OnboardingRequest, db_session: Session +) -> list[SourceStatus]: + """Per-cc_pair scrape status for the request (from the latest IndexAttempt).""" + statuses: list[SourceStatus] = [] + for cc_pair_id in request.cc_pair_ids or []: + cc_pair = get_connector_credential_pair_from_id(cc_pair_id, db_session) + if cc_pair is None: + continue + latest = db_session.execute( + select(IndexAttempt) + .where(IndexAttempt.connector_id == cc_pair.connector_id) + .where(IndexAttempt.credential_id == cc_pair.credential_id) + .order_by(desc(IndexAttempt.time_created)) + .limit(1) + ).scalar_one_or_none() + statuses.append( + SourceStatus( + cc_pair_id=cc_pair_id, + name=cc_pair.name, + status=latest.status.value if latest else "not_started", + docs_indexed=(latest.total_docs_indexed or 0) if latest else 0, + error_msg=latest.error_msg if latest else None, + ) + ) + return statuses + + +@basic_router.get("/{request_id}/status") +def onboarding_status( + request_id: int, + user: User | None = Depends(current_user), + db_session: Session = Depends(get_session), +) -> OnboardingStatusResponse: + _require_user(user) + request = get_onboarding_request(db_session, request_id) + if request is None: + raise HTTPException(status_code=404, detail="Onboarding request not found.") + # Requester or an admin may view. + if ( + user is not None + and user.role != UserRole.ADMIN + and request.requester_id != user.id + ): + raise HTTPException(status_code=403, detail="Not your onboarding request.") + + sources = _source_statuses(request, db_session) + # Auto-advance INDEXING -> COMPLETE once every source has indexed successfully. + if ( + request.status == OnboardingStatus.INDEXING.value + and sources + and all(s.status == IndexingStatus.SUCCESS.value for s in sources) + ): + update_onboarding_status(db_session, request, OnboardingStatus.COMPLETE) + return OnboardingStatusResponse( + request_id=request.id, status=request.status, sources=sources + ) + + +# --- admin approval --------------------------------------------------------- + + +@admin_router.get("") +def list_requests( + _: User | None = Depends(current_admin_user), + db_session: Session = Depends(get_session), +) -> list[OnboardingRequestSnapshot]: + return [ + OnboardingRequestSnapshot.from_model(r) + for r in list_onboarding_requests(db_session) + ] + + +@admin_router.post("/{request_id}/approve") +def approve_request( + request_id: int, + admin: User | None = Depends(current_admin_user), + db_session: Session = Depends(get_session), +) -> OnboardingRequestSnapshot: + request = get_onboarding_request(db_session, request_id) + if request is None: + raise HTTPException(status_code=404, detail="Onboarding request not found.") + if request.status != OnboardingStatus.PENDING.value: + raise HTTPException( + status_code=400, + detail=f"Request is '{request.status}', only pending requests can be approved.", + ) + if admin is None: + raise HTTPException( + status_code=403, detail="Admin identity required to approve." + ) + try: + provision_onboarding(request, admin, db_session) + except Exception as e: + # provision_onboarding already marked the request FAILED; surface the error. + raise HTTPException(status_code=500, detail=f"Provisioning failed: {e}") + return OnboardingRequestSnapshot.from_model(request) + + +@admin_router.post("/{request_id}/reject") +def reject_request( + request_id: int, + decision: DecisionRequest, + admin: User | None = Depends(current_admin_user), + db_session: Session = Depends(get_session), +) -> OnboardingRequestSnapshot: + request = get_onboarding_request(db_session, request_id) + if request is None: + raise HTTPException(status_code=404, detail="Onboarding request not found.") + if request.status != OnboardingStatus.PENDING.value: + raise HTTPException( + status_code=400, + detail=f"Request is '{request.status}', only pending requests can be rejected.", + ) + update_onboarding_status( + db_session, + request, + OnboardingStatus.REJECTED, + approver_id=admin.id if admin else None, + decision_reason=decision.reason, + ) + return OnboardingRequestSnapshot.from_model(request) diff --git a/backend/danswer/server/features/onboarding/models.py b/backend/danswer/server/features/onboarding/models.py new file mode 100644 index 00000000000..8594121038f --- /dev/null +++ b/backend/danswer/server/features/onboarding/models.py @@ -0,0 +1,107 @@ +"""Request/response models for the self-serve onboarding API.""" +import datetime +from typing import Literal + +from pydantic import BaseModel + +from danswer.db.models import OnboardingRequest + + +class ChannelRef(BaseModel): + channel_id: str + channel_name: str + + +class OnboardingSourceModel(BaseModel): + # Priority is the order in the sources list (index 0 = highest). + type: Literal["web", "confluence", "github", "slack"] + value: str # docs root URL / confluence URL / github repo URL / slack channel name + label: str | None = None + + +class SmeOption(BaseModel): + enabled: bool = False + group_name: str = "" # comma-separated Slack user-group names/@handles + + +class OncallOption(BaseModel): + enabled: bool = False + schedule: str = "" # opsgenie schedule name to tag on "need more help" + + +class JiraOption(BaseModel): + enabled: bool = False + project_key: str = "" + issue_type: str = "" + component: str = "" + + +class OnboardingSubmitRequest(BaseModel): + team_name: str + channel: ChannelRef + response_type: Literal["citations", "quotes"] = "citations" + respond_tag_only: bool = False + system_prompt: str = "" + task_prompt: str = "" + sme: SmeOption = SmeOption() + oncall: OncallOption = OncallOption() + jira: JiraOption = JiraOption() + sources: list[OnboardingSourceModel] + + def to_payload(self) -> dict: + return self.dict() + + +class OnboardingRequestSnapshot(BaseModel): + id: int + requester_email: str + status: str + payload: dict + decision_reason: str | None + error_msg: str | None + persona_id: int | None + document_set_id: int | None + slack_bot_config_id: int | None + cc_pair_ids: list[int] | None + created_at: datetime.datetime + updated_at: datetime.datetime + + @classmethod + def from_model(cls, r: OnboardingRequest) -> "OnboardingRequestSnapshot": + return cls( + id=r.id, + requester_email=r.requester_email, + status=r.status, + payload=r.payload, + decision_reason=r.decision_reason, + error_msg=r.error_msg, + persona_id=r.persona_id, + document_set_id=r.document_set_id, + slack_bot_config_id=r.slack_bot_config_id, + cc_pair_ids=r.cc_pair_ids, + created_at=r.created_at, + updated_at=r.updated_at, + ) + + +class ValidateRequest(BaseModel): + kind: Literal["slack_channel", "slack_group", "confluence", "docs"] + value: str + + +class DecisionRequest(BaseModel): + reason: str | None = None + + +class SourceStatus(BaseModel): + cc_pair_id: int + name: str + status: str # IndexingStatus value, or "not_started" + docs_indexed: int + error_msg: str | None + + +class OnboardingStatusResponse(BaseModel): + request_id: int + status: str + sources: list[SourceStatus] diff --git a/backend/tests/unit/danswer/onboarding/__init__.py b/backend/tests/unit/danswer/onboarding/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/backend/tests/unit/danswer/onboarding/test_provision.py b/backend/tests/unit/danswer/onboarding/test_provision.py new file mode 100644 index 00000000000..ccba425a006 --- /dev/null +++ b/backend/tests/unit/danswer/onboarding/test_provision.py @@ -0,0 +1,115 @@ +"""Unit tests for the onboarding provisioning builders (pure logic — no DB/IO).""" +from danswer.configs.constants import DocumentSource +from danswer.onboarding.provision import _build_channel_config +from danswer.onboarding.provision import _build_connector_base +from danswer.onboarding.provision import _parse_github_repo +from danswer.onboarding.provision import _source_type_value + + +def test_web_docs_uipath_enables_all_versions() -> None: + cb = _build_connector_base( + { + "type": "web", + "value": "https://docs.uipath.com/orchestrator/automation-cloud/latest", + }, + db_session=None, # type: ignore[arg-type] # web path doesn't touch the db + ) + assert cb.source == DocumentSource.WEB + cfg = cb.connector_specific_config + assert cfg["base_url"].startswith("https://docs.uipath.com/orchestrator") + assert cfg["uipath_latest_versions"] is True + assert cfg["web_connector_type"] == "recursive" + + +def test_web_non_docs_does_not_enable_all_versions() -> None: + cb = _build_connector_base( + {"type": "web", "value": "https://example.com/help"}, + db_session=None, # type: ignore[arg-type] + ) + assert "uipath_latest_versions" not in cb.connector_specific_config + + +def test_confluence_source_config() -> None: + url = "https://uipath.atlassian.net/wiki/spaces/DEV/overview" + cb = _build_connector_base( + {"type": "confluence", "value": url}, db_session=None # type: ignore[arg-type] + ) + assert cb.source == DocumentSource.CONFLUENCE + assert cb.connector_specific_config["wiki_page_url"] == url + + +def test_github_source_config_parses_owner_repo() -> None: + cb = _build_connector_base( + {"type": "github", "value": "https://github.com/UiPath/testsfagents"}, + db_session=None, # type: ignore[arg-type] + ) + assert cb.source == DocumentSource.GITHUB + assert cb.connector_specific_config["repo_owner"] == "UiPath" + assert cb.connector_specific_config["repo_name"] == "testsfagents" + + +def test_parse_github_repo() -> None: + assert _parse_github_repo("https://github.com/UiPath/danswer") == ( + "UiPath", + "danswer", + ) + + +def test_source_type_value_mapping() -> None: + assert _source_type_value("web") == DocumentSource.WEB.value + assert _source_type_value("confluence") == DocumentSource.CONFLUENCE.value + assert _source_type_value("github") == DocumentSource.GITHUB.value + assert _source_type_value("slack") == DocumentSource.SLACK.value + + +def _payload(**over: object) -> dict: + base = { + "team_name": "Team X", + "channel": {"channel_id": "C1", "channel_name": "help-x"}, + "respond_tag_only": True, + "sme": {"enabled": False, "group_name": ""}, + "oncall": {"enabled": False, "schedule": ""}, + "jira": {"enabled": False}, + } + base.update(over) + return base + + +def test_channel_config_basic() -> None: + cfg = _build_channel_config(_payload(), prioritized_sources=["confluence", "web"]) + assert cfg["channel_names"] == ["help-x"] + assert cfg["respond_tag_only"] is True + assert cfg["prioritized_sources"] == ["confluence", "web"] + assert "enable_sme_validation" not in cfg + assert "opsgenie_schedule" not in cfg + assert "jira_config" not in cfg + + +def test_channel_config_all_options() -> None: + cfg = _build_channel_config( + _payload( + sme={"enabled": True, "group_name": "as-smes"}, + oncall={"enabled": True, "schedule": "AS-OnCall"}, + jira={ + "enabled": True, + "project_key": "AS", + "issue_type": "Bug", + "component": "core", + }, + ), + prioritized_sources=["web"], + ) + assert cfg["enable_sme_validation"] is True + assert cfg["sme_group_name"] == "as-smes" + assert cfg["opsgenie_schedule"] == "AS-OnCall" + assert cfg["jira_config"]["enable_jira_integration"] is True + assert cfg["jira_config"]["project_key"] == "AS" + + +def test_channel_config_oncall_enabled_but_no_schedule_omitted() -> None: + # Enabling on-call without a schedule shouldn't write an empty opsgenie value. + cfg = _build_channel_config( + _payload(oncall={"enabled": True, "schedule": ""}), + prioritized_sources=[], + ) + assert "opsgenie_schedule" not in cfg diff --git a/backend/tests/unit/danswer/onboarding/test_validation.py b/backend/tests/unit/danswer/onboarding/test_validation.py new file mode 100644 index 00000000000..39d0172caf8 --- /dev/null +++ b/backend/tests/unit/danswer/onboarding/test_validation.py @@ -0,0 +1,31 @@ +"""Unit tests for onboarding docs-URL validation (pure — no network). + +Slack/Confluence validators hit live APIs and are covered by manual/integration +checks, not here.""" +from danswer.onboarding.validation import validate_docs_url + + +def test_docs_url_valid_root_normalizes() -> None: + r = validate_docs_url( + "https://docs.uipath.com/orchestrator/automation-cloud/latest/user-guide/x" + ) + assert r.valid + # Normalized to the product root (version/'latest' segment stripped). + assert ( + r.resolved["root_url"] + == "https://docs.uipath.com/orchestrator/automation-cloud" + ) + + +def test_docs_url_rejects_non_docs_domain() -> None: + r = validate_docs_url("https://example.com/foo") + assert not r.valid + + +def test_docs_url_requires_product_path() -> None: + r = validate_docs_url("https://docs.uipath.com/") + assert not r.valid + + +def test_docs_url_empty() -> None: + assert not validate_docs_url("").valid diff --git a/web/src/app/admin/onboarding/OnboardingRequestsTable.tsx b/web/src/app/admin/onboarding/OnboardingRequestsTable.tsx new file mode 100644 index 00000000000..828f751a58d --- /dev/null +++ b/web/src/app/admin/onboarding/OnboardingRequestsTable.tsx @@ -0,0 +1,169 @@ +"use client"; + +import { useEffect, useState } from "react"; +import { + OnboardingRequestSnapshot, + OnboardingStatusResponse, +} from "@/lib/onboarding/interfaces"; + +function Badge({ status }: { status: string }) { + const color = + status === "complete" + ? "text-link" + : status === "failed" || status === "rejected" + ? "text-error" + : status === "pending" + ? "text-default" + : "text-subtle"; + return ( + {status} + ); +} + +export function OnboardingRequestsTable() { + const [requests, setRequests] = useState([]); + const [statuses, setStatuses] = useState< + Record + >({}); + const [busy, setBusy] = useState(null); + const [error, setError] = useState(null); + + async function refresh() { + const r = await fetch("/api/admin/onboarding"); + if (r.ok) setRequests((await r.json()) as OnboardingRequestSnapshot[]); + } + + useEffect(() => { + void refresh(); + }, []); + + async function decide(id: number, action: "approve" | "reject") { + setError(null); + let reason: string | null = null; + if (action === "reject") { + reason = window.prompt("Reason for rejecting (optional)?") || null; + } + setBusy(id); + try { + const r = await fetch(`/api/admin/onboarding/${id}/${action}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(action === "reject" ? { reason } : {}), + }); + if (!r.ok) { + setError( + (await r.json().catch(() => null))?.detail || `Failed (${r.status}).` + ); + return; + } + await refresh(); + } finally { + setBusy(null); + } + } + + async function loadStatus(id: number) { + const r = await fetch(`/api/onboarding/${id}/status`); + if (r.ok) { + const data = (await r.json()) as OnboardingStatusResponse; + setStatuses((p) => ({ ...p, [id]: data })); + } + } + + if (requests.length === 0) { + return

No onboarding requests yet.

; + } + + return ( +
+ {error &&

{error}

} + {requests.map((req) => ( +
+
+
+
+ {req.payload.team_name} → #{req.payload.channel.channel_name} +
+
+ by {req.requester_email} ·{" "} + {new Date(req.created_at).toLocaleString()} +
+
+ +
+ +
+ Sources:{" "} + {req.payload.sources + .map((s) => `${s.type}:${s.label || s.value}`) + .join(", ")} +
+
+ SME:{" "} + {req.payload.sme.enabled + ? req.payload.sme.group_name || "yes" + : "no"}{" "} + · On-call:{" "} + {req.payload.oncall.enabled + ? req.payload.oncall.schedule || "yes" + : "no"}{" "} + · Jira:{" "} + {req.payload.jira.enabled + ? req.payload.jira.project_key || "yes" + : "no"} +
+ {req.error_msg && ( +

{req.error_msg}

+ )} + + {req.status === "pending" && ( +
+ + +
+ )} + + {(req.cc_pair_ids?.length ?? 0) > 0 && ( +
+ + {statuses[req.id]?.sources.map((s) => ( +
+ {s.name} + + · {s.docs_indexed} docs + {s.error_msg ? ( + · {s.error_msg} + ) : null} + +
+ ))} +
+ )} +
+ ))} +
+ ); +} diff --git a/web/src/app/admin/onboarding/page.tsx b/web/src/app/admin/onboarding/page.tsx new file mode 100644 index 00000000000..e424a6e13df --- /dev/null +++ b/web/src/app/admin/onboarding/page.tsx @@ -0,0 +1,19 @@ +import { AdminPageTitle } from "@/components/admin/Title"; +import { RobotIcon } from "@/components/icons/icons"; +import { OnboardingRequestsTable } from "./OnboardingRequestsTable"; + +export default function Page() { + return ( +
+ } + title="Onboarding requests" + /> +

+ Approve a request to auto-provision the team's assistant, sources + (scraped with bumped priority), and Slack bot config. +

+ +
+ ); +} diff --git a/web/src/app/onboarding/OnboardingForm.tsx b/web/src/app/onboarding/OnboardingForm.tsx new file mode 100644 index 00000000000..773cc72d1b7 --- /dev/null +++ b/web/src/app/onboarding/OnboardingForm.tsx @@ -0,0 +1,579 @@ +"use client"; + +import { useEffect, useState } from "react"; +import { + OnboardingRequestSnapshot, + OnboardingSource, + OnboardingSourceType, + OnboardingStatusResponse, + validateOnboardingField, + ValidateKind, + ValidationResult, +} from "@/lib/onboarding/interfaces"; + +// --- inline-validated text field -------------------------------------------- + +function ValidatedField({ + label, + placeholder, + kind, + value, + onChange, + onResolved, + optional, +}: { + label: string; + placeholder?: string; + kind: ValidateKind; + value: string; + onChange: (v: string) => void; + onResolved?: (r: ValidationResult) => void; + optional?: boolean; +}) { + const [result, setResult] = useState(null); + const [checking, setChecking] = useState(false); + + async function check() { + if (!value.trim()) { + setResult(null); + return; + } + setChecking(true); + const r = await validateOnboardingField(kind, value); + setResult(r); + setChecking(false); + onResolved?.(r); + } + + return ( +
+ + { + onChange(e.target.value); + setResult(null); + }} + onBlur={check} + className="w-full rounded-md border border-border-medium bg-background-weak px-3 py-2 text-sm text-default focus:outline-none focus:ring-1 focus:ring-accent" + /> + {checking &&

Validating…

} + {result && ( +

+ {result.valid ? "✓ " : "✗ "} + {result.message} +

+ )} +
+ ); +} + +// --- status badge ----------------------------------------------------------- + +function StatusBadge({ status }: { status: string }) { + const color = + status === "complete" + ? "text-link" + : status === "failed" || status === "rejected" + ? "text-error" + : "text-subtle"; + return {status}; +} + +// --- one extra source row --------------------------------------------------- + +function SourceRow({ + source, + onChange, + onRemove, + onMove, +}: { + source: OnboardingSource; + onChange: (s: OnboardingSource) => void; + onRemove: () => void; + onMove: (dir: -1 | 1) => void; +}) { + const kindFor: Record = { + web: "docs", + confluence: "confluence", + github: "docs", // github URLs aren't live-validated; format-check only + slack: "slack_channel", + }; + const [result, setResult] = useState(null); + return ( +
+ +
+ { + onChange({ ...source, value: e.target.value }); + setResult(null); + }} + onBlur={async () => { + if (!source.value.trim() || source.type === "github") return; + setResult( + await validateOnboardingField(kindFor[source.type], source.value) + ); + }} + className="w-full rounded-md border border-border-medium bg-background-weak px-3 py-2 text-sm" + /> + {result && ( +

+ {result.valid ? "✓ " : "✗ "} + {result.message} +

+ )} +
+ + + +
+ ); +} + +// --- main form -------------------------------------------------------------- + +export function OnboardingForm() { + const [teamName, setTeamName] = useState(""); + const [channelInput, setChannelInput] = useState(""); + const [channel, setChannel] = useState<{ id: string; name: string } | null>( + null + ); + const [responseType, setResponseType] = useState<"citations" | "quotes">( + "citations" + ); + const [respondTagOnly, setRespondTagOnly] = useState(false); + const [systemPrompt, setSystemPrompt] = useState(""); + const [smeEnabled, setSmeEnabled] = useState(false); + const [smeGroup, setSmeGroup] = useState(""); + const [oncallEnabled, setOncallEnabled] = useState(false); + const [oncallSchedule, setOncallSchedule] = useState(""); + const [jiraEnabled, setJiraEnabled] = useState(false); + const [jiraProject, setJiraProject] = useState(""); + const [jiraIssueType, setJiraIssueType] = useState(""); + + const [docsCloud, setDocsCloud] = useState(""); + const [docsOnprem, setDocsOnprem] = useState(""); + const [includeHistory, setIncludeHistory] = useState(true); + const [extraSources, setExtraSources] = useState([]); + + const [submitting, setSubmitting] = useState(false); + const [error, setError] = useState(null); + const [mine, setMine] = useState([]); + const [statuses, setStatuses] = useState< + Record + >({}); + + useEffect(() => { + fetch("/api/onboarding/default-prompt") + .then((r) => (r.ok ? r.json() : null)) + .then((d) => d?.system_prompt && setSystemPrompt(d.system_prompt)) + .catch(() => {}); + void refreshMine(); + }, []); + + async function refreshMine() { + try { + const r = await fetch("/api/onboarding/mine"); + if (r.ok) setMine((await r.json()) as OnboardingRequestSnapshot[]); + } catch { + /* ignore */ + } + } + + async function loadStatus(id: number) { + const r = await fetch(`/api/onboarding/${id}/status`); + if (r.ok) { + const data = (await r.json()) as OnboardingStatusResponse; + setStatuses((prev) => ({ ...prev, [id]: data })); + } + } + + function buildSources(): OnboardingSource[] { + const s: OnboardingSource[] = []; + if (docsCloud.trim()) + s.push({ type: "web", value: docsCloud.trim(), label: "Docs (cloud)" }); + if (docsOnprem.trim()) + s.push({ + type: "web", + value: docsOnprem.trim(), + label: "Docs (on-prem)", + }); + if (includeHistory && channel) + s.push({ + type: "slack", + value: channel.name, + label: `#${channel.name} history`, + }); + extraSources.forEach((x) => x.value.trim() && s.push(x)); + return s; + } + + async function submit() { + setError(null); + if (!teamName.trim()) return setError("Team name is required."); + if (!channel) return setError("Validate the bot channel first."); + const sources = buildSources(); + if (sources.length === 0) return setError("Add at least one source."); + + setSubmitting(true); + try { + const res = await fetch("/api/onboarding", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + team_name: teamName.trim(), + channel: { channel_id: channel.id, channel_name: channel.name }, + response_type: responseType, + respond_tag_only: respondTagOnly, + system_prompt: systemPrompt, + task_prompt: "", + sme: { enabled: smeEnabled, group_name: smeGroup }, + oncall: { enabled: oncallEnabled, schedule: oncallSchedule }, + jira: { + enabled: jiraEnabled, + project_key: jiraProject, + issue_type: jiraIssueType, + component: "", + }, + sources, + }), + }); + if (!res.ok) { + setError( + (await res.json().catch(() => null))?.detail || + `Submit failed (${res.status}).` + ); + return; + } + // reset the source fields; keep it simple. + setDocsCloud(""); + setDocsOnprem(""); + setExtraSources([]); + await refreshMine(); + } catch { + setError("Something went wrong submitting the request."); + } finally { + setSubmitting(false); + } + } + + const section = + "mb-6 rounded-lg border border-border-medium bg-background-weak p-4"; + + return ( +
+

+ Onboard a team to Darwin +

+

+ Submit this request; an admin approves it, then Darwin auto-scrapes your + sources and wires up your assistant. Every field is validated live. +

+ +
+ + setChannel( + r.valid + ? { + id: String(r.resolved.channel_id ?? ""), + name: String(r.resolved.channel_name ?? ""), + } + : null + ) + } + /> + + setTeamName(e.target.value)} + placeholder="e.g. Integration Service" + className="w-full rounded-md border border-border-medium bg-background-weak px-3 py-2 text-sm mb-3" + /> +
+ + +
+
+ +
+

+ Sources (priority order) +

+ + +

+ Paste the product root URL only — Darwin crawls all versions + automatically. +

+ +
+ Additional sources +
+ {extraSources.map((s, i) => ( + + setExtraSources((p) => p.map((x, j) => (j === i ? ns : x))) + } + onRemove={() => setExtraSources((p) => p.filter((_, j) => j !== i))} + onMove={(dir) => + setExtraSources((p) => { + const j = i + dir; + if (j < 0 || j >= p.length) return p; + const c = [...p]; + [c[i], c[j]] = [c[j], c[i]]; + return c; + }) + } + /> + ))} + +
+ +
+ +

+ Prefilled from the default (Orchestrator) — edit as needed. +

+