diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e26f52a..dcbdf25 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -618,7 +618,7 @@ jobs: [**Android**](https://central.sonatype.com/artifact/ai.sqlite/sync): `ai.sqlite:sync:${{ steps.tag.outputs.version }}` [**Swift**](https://github.com/sqliteai/sqlite-sync#swift-package): [Installation Guide](https://github.com/sqliteai/sqlite-sync#swift-package) [**Docker (PostgreSQL)**](https://hub.docker.com/r/sqlitecloud/sqlite-sync-postgres): `docker pull sqlitecloud/sqlite-sync-postgres:17` or `:15` - [**Docker (Supabase)**](https://hub.docker.com/r/sqlitecloud/sqlite-sync-supabase): `docker pull sqlitecloud/sqlite-sync-supabase:17` or `:15` + [**Docker (Supabase)**](https://hub.docker.com/r/sqlitecloud/sqlite-sync-supabase): `docker pull sqlitecloud/sqlite-sync-supabase:17`, `:15` or `:17-alpine` --- @@ -631,7 +631,7 @@ jobs: docker-publish: runs-on: ubuntu-22.04 - name: docker ${{ matrix.image }} pg${{ matrix.pg_major }}${{ github.ref != 'refs/heads/main' && ' beta' || '' }} + name: docker ${{ matrix.image }} pg${{ matrix.pg_major }}${{ matrix.tag_suffix }}${{ github.ref != 'refs/heads/main' && ' beta' || '' }} needs: [build, postgres-test, postgres-build, release] # main: publish the versioned release tags, only when the release job cut a new # release. Any other branch: release is skipped, publish/overwrite a per-branch @@ -659,7 +659,16 @@ jobs: - image: sqlite-sync-supabase pg_major: '15' dockerfile: docker/postgresql/Dockerfile.supabase.release - supabase_tag: '15.8.1.085' + supabase_tag: '15.8.1.135' + # Newer Supabase 17 images use an Alpine userland (still a glibc Nix + # postgres, so the same glibc artifact is installed). Published behind + # explicit -alpine / exact-base tags; the shared :17 alias stays on the + # Ubuntu build until we decide the alias scheme. + - image: sqlite-sync-supabase + pg_major: '17' + dockerfile: docker/postgresql/Dockerfile.supabase.release + supabase_tag: '17.6.1.151' + tag_suffix: '-alpine' steps: @@ -729,12 +738,15 @@ jobs: VERSION=${{ steps.version.outputs.version }} IMAGE=${{ env.DOCKERHUB_ORG }}/${{ matrix.image }} SUPABASE_TAG=${{ matrix.supabase_tag }} + # Empty for the default (Ubuntu) rows; "-alpine" for the Alpine variant. + # Keeps the Alpine images from colliding with the : aliases. + SUFFIX="${{ matrix.tag_suffix }}" BETA="${{ steps.beta.outputs.slug }}" { if [ -n "$BETA" ]; then - echo "tags=${IMAGE}:${{ matrix.pg_major }}-beta-${BETA}" + echo "tags=${IMAGE}:${{ matrix.pg_major }}${SUFFIX}-beta-${BETA}" else - echo "tags=${IMAGE}:${{ matrix.pg_major }},${IMAGE}:${{ matrix.pg_major }}-${VERSION},${IMAGE}:${SUPABASE_TAG}" + echo "tags=${IMAGE}:${{ matrix.pg_major }}${SUFFIX},${IMAGE}:${{ matrix.pg_major }}${SUFFIX}-${VERSION},${IMAGE}:${SUPABASE_TAG}" fi echo "build_args<> $GITHUB_OUTPUT + # Build amd64 natively first and load it into the daemon, so the extension + # is proven to load before anything is published. The multi-arch push below + # reuses these layers from the builder cache. + - name: build amd64 image for the smoke test + uses: docker/build-push-action@v6 + with: + context: . + file: ${{ matrix.dockerfile }} + platforms: linux/amd64 + load: true + tags: cloudsync-smoke:test + build-args: ${{ matrix.image == 'sqlite-sync-postgres' && steps.standalone.outputs.build_args || steps.supabase.outputs.build_args }} + + - name: smoke test the extension + env: + # the supabase bases ship a `postgres` role that cannot create the + # extension; their own docs use supabase_admin + PSQL_USER: ${{ matrix.image == 'sqlite-sync-supabase' && 'supabase_admin' || 'postgres' }} + run: | + docker run -d --name cloudsync-smoke -e POSTGRES_PASSWORD=postgres cloudsync-smoke:test + # the entrypoint's init-time server listens on the unix socket only, so + # a TCP connection only succeeds once the real server is up + ready='' + for _ in $(seq 1 60); do + if docker exec -e PGPASSWORD=postgres cloudsync-smoke \ + psql -h 127.0.0.1 -U "$PSQL_USER" -d postgres -tAc 'SELECT 1' >/dev/null 2>&1; then + ready=1 + break + fi + sleep 2 + done + if [ -z "$ready" ]; then + echo "::error::postgres never accepted TCP connections" + docker logs cloudsync-smoke + exit 1 + fi + if ! docker exec -e PGPASSWORD=postgres cloudsync-smoke \ + psql -h 127.0.0.1 -U "$PSQL_USER" -d postgres -v ON_ERROR_STOP=1 \ + -c 'CREATE EXTENSION IF NOT EXISTS cloudsync;' \ + -c 'SELECT cloudsync_version();'; then + echo "::error::cloudsync failed to load in ${{ matrix.image }} pg${{ matrix.pg_major }}${{ matrix.tag_suffix }}" + docker logs cloudsync-smoke + exit 1 + fi + docker rm -f cloudsync-smoke + - uses: docker/build-push-action@v6 with: context: . diff --git a/.gitignore b/.gitignore index 355141b..167c773 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,9 @@ openssl/ unittest .env +# Generated Supabase secrets, if a deployment .env is ever pulled down locally +/deploy/supabase-flyio/.env* + # Node.js **/node_modules/** package-lock.json diff --git a/CHANGELOG.md b/CHANGELOG.md index d1a9aeb..8d260b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [Unreleased] + +### Added + +- **`sqlitecloud/sqlite-sync-supabase:17-alpine`** — CloudSync on the newer Alpine-userland Supabase Postgres images (roughly `17.6.1.084` and later). The extension is identical to the Ubuntu build — the PostgreSQL binary is glibc-linked on both — so only the base image differs. Use it when your Supabase stack pins one of these newer bases; `:17` continues to track the Ubuntu base. + +### Changed + +- **The `sqlitecloud/sqlite-sync-supabase:15` image now builds on Supabase base `15.8.1.135`** (previously `15.8.1.085`), which moves it from Ubuntu 20.04 to 24.04. Ubuntu 24.04 allocates system UIDs differently, so the `postgres` user changes from `105:106` to `101:102` and an **existing** data directory becomes unreadable: the container fails with `cat: /etc/postgresql-custom/pgsodium_root.key: Permission denied` followed by `FATAL: invalid secret key`. Existing deployments need a one-time `chown -R 101:102` on the data directory and the `supabase_db-config` volume before starting the new image — see [Self-Hosted Supabase](docs/postgresql/quickstarts/supabase-self-hosted.md) for the procedure. New deployments are unaffected, and the CloudSync extension itself is unchanged. + ## [1.1.2] - 2026-07-13 ### Fixed diff --git a/deploy/supabase-flyio/Dockerfile b/deploy/supabase-flyio/Dockerfile new file mode 100644 index 0000000..6c44d95 --- /dev/null +++ b/deploy/supabase-flyio/Dockerfile @@ -0,0 +1,23 @@ +# Self-hosted Supabase + CloudSync, packaged as a single Fly Machine image. +# +# The image ships dockerd + docker compose and a pinned copy of the upstream +# Supabase `docker/` directory. Everything the stack needs at runtime (images, +# PGDATA, .env) lives on the Fly volume mounted at /data, so machine restarts +# and redeploys keep the data and re-run the same bootstrap. +FROM docker:28-dind + +# fuse-overlayfs: Docker's overlay2 driver fails inside a Fly Machine +# ("failed to convert whiteout file: operation not permitted"). +RUN apk add --no-cache bash fuse-overlayfs openssl + +# Pinned upstream Supabase docker/ tree. Bump to adopt new service versions. +ARG SUPABASE_REF=312d05af4be0a404325417254d57f69378a0091c +RUN wget -qO /tmp/sb.tgz "https://codeload.github.com/supabase/supabase/tar.gz/${SUPABASE_REF}" \ + && tar -xzf /tmp/sb.tgz -C /tmp \ + && mv /tmp/supabase-*/docker /supabase \ + && rm -rf /tmp/sb.tgz /tmp/supabase-* + +COPY docker-compose.cloudsync.yml cloudsync.sql entrypoint.sh /supabase/ +RUN chmod +x /supabase/entrypoint.sh + +ENTRYPOINT ["/supabase/entrypoint.sh"] diff --git a/deploy/supabase-flyio/cloudsync.sql b/deploy/supabase-flyio/cloudsync.sql new file mode 100644 index 0000000..c1b24be --- /dev/null +++ b/deploy/supabase-flyio/cloudsync.sql @@ -0,0 +1 @@ +CREATE EXTENSION IF NOT EXISTS cloudsync; diff --git a/deploy/supabase-flyio/docker-compose.cloudsync.yml b/deploy/supabase-flyio/docker-compose.cloudsync.yml new file mode 100644 index 0000000..dd69e33 --- /dev/null +++ b/deploy/supabase-flyio/docker-compose.cloudsync.yml @@ -0,0 +1,17 @@ +# Layered on top of the upstream docker-compose.yml via COMPOSE_FILE. +# Keeps the upstream file untouched so it can be re-pinned without merge work. +services: + db: + # Released tag by default; set CLOUDSYNC_IMAGE (e.g. `fly secrets set`) to + # run a beta build without touching the repo. + image: ${CLOUDSYNC_IMAGE:-sqlitecloud/sqlite-sync-supabase:17-alpine} + volumes: + - ./cloudsync.sql:/docker-entrypoint-initdb.d/init-scripts/100-cloudsync.sql:Z + ports: + # Direct Postgres, bypassing Supavisor (which already owns 5432). + - "5433:5432" + + supavisor: + # Skips /app/limits.sh, whose `ulimit -n 100000` is refused by the Fly kernel + # and crashes the container. Same tini init, same command. + entrypoint: ["/usr/bin/tini", "-s", "-g", "--"] diff --git a/deploy/supabase-flyio/entrypoint.sh b/deploy/supabase-flyio/entrypoint.sh new file mode 100644 index 0000000..44c6125 --- /dev/null +++ b/deploy/supabase-flyio/entrypoint.sh @@ -0,0 +1,103 @@ +#!/bin/sh +# Boots the whole stack. Idempotent: runs on every machine start, including +# after `fly deploy`, `fly machine restart` and crashes. +set -eu + +DATA_DIR=/data/supabase-docker +DOCKERD_LOG=/data/dockerd.log +INIT_MARKER=/data/.cloudsync-post-init-done + +echo "=== configuring dockerd ===" +mkdir -p /etc/docker /data/docker +cat > /etc/docker/daemon.json <<'JSON' +{ + "storage-driver": "fuse-overlayfs", + "data-root": "/data/docker", + "ipv6": true, + "ip6tables": true, + "fixed-cidr-v6": "fd00:d0c::/64" +} +JSON + +echo "=== starting dockerd ===" +dockerd >>"$DOCKERD_LOG" 2>&1 & +until docker info >/dev/null 2>&1; do sleep 1; done + +echo "=== syncing compose tree to $DATA_DIR ===" +mkdir -p "$DATA_DIR" +# Config comes from the image on every boot; state (.env, volumes/) stays. +cp -R /supabase/. "$DATA_DIR/" +cd "$DATA_DIR" + +set_env() { + grep -q "^$1=" .env && sed -i "s|^$1=.*|$1=$2|" .env || echo "$1=$2" >> .env +} + +if [ ! -f .env ]; then + echo "=== first boot: generating secrets ===" + cp .env.example .env + sh ./utils/generate-keys.sh --update-env + sh ./utils/add-new-auth-keys.sh --update-env + set_env POSTGRES_PASSWORD "$(openssl rand -hex 24)" + set_env DASHBOARD_PASSWORD "$(openssl rand -hex 12)" +fi + +# Public hostnames follow from the app name, which Fly injects, so no +# deployment-specific URL has to live in the repo. Fly secrets still win. +if [ -n "${FLY_APP_NAME:-}" ]; then + : "${SUPABASE_PUBLIC_URL:=https://$FLY_APP_NAME.fly.dev}" + : "${API_EXTERNAL_URL:=$SUPABASE_PUBLIC_URL/auth/v1}" + export SUPABASE_PUBLIC_URL API_EXTERNAL_URL +fi + +# Services dropped from the active config — a gateway swapped out via an override +# profile, say — keep their containers and their published ports, which blocks the +# replacement from binding. `--remove-orphans` does not cover them. +active_services=$(docker compose config --services 2>/dev/null || true) +docker compose ps -a --format '{{.Service}}|{{.Name}}' 2>/dev/null | while IFS='|' read -r svc name; do + [ -n "$svc" ] || continue + echo "$active_services" | grep -qx "$svc" && continue + echo "=== removing disabled service: $svc ($name) ===" + docker rm -f "$name" >/dev/null 2>&1 || true +done + +echo "=== starting supabase ===" +# A service that fails its dependency conditions must not kill the machine: +# exiting here would crash-loop until Fly gives up, taking the healthy services +# with it and leaving nothing to debug. +docker compose up -d --remove-orphans || echo "WARNING: compose up reported a failure, continuing" + +echo "=== waiting for postgres ===" +pg_ready=0 +for _ in $(seq 1 90); do + if docker compose exec -T db pg_isready -U postgres -h localhost >/dev/null 2>&1; then + pg_ready=1 + break + fi + sleep 2 +done + +if [ "$pg_ready" = 1 ]; then + # Init scripts only run on an empty PGDATA, so create the extension explicitly. + docker compose exec -T db psql -U postgres -c "CREATE EXTENSION IF NOT EXISTS cloudsync;" || true +else + echo "WARNING: postgres never became ready, skipping post-init" +fi + +if [ "$pg_ready" = 1 ] && [ ! -f "$INIT_MARKER" ]; then + # GoTrue runs its migrations as supabase_auth_admin and must own these. + echo "=== fixing auth function ownership ===" + for fn in "auth.uid()" "auth.role()" "auth.email()"; do + docker compose exec -T db psql -U postgres \ + -c "ALTER FUNCTION $fn OWNER TO supabase_auth_admin;" || true + done + docker compose restart auth + touch "$INIT_MARKER" +fi + +echo "=== supabase is up ===" +docker compose ps + +# Stream service logs to `fly logs`. If this exits the machine restarts and the +# whole bootstrap re-runs. +exec docker compose logs --no-color -f diff --git a/deploy/supabase-flyio/fly.toml b/deploy/supabase-flyio/fly.toml new file mode 100644 index 0000000..069f3fc --- /dev/null +++ b/deploy/supabase-flyio/fly.toml @@ -0,0 +1,43 @@ +# No `app` / `primary_region` here on purpose: both are deployment-specific and +# are passed on the command line instead. +# +# fly deploy . --config fly.toml --app "$FLY_APP" --primary-region "$FLY_REGION" +# +# The public URLs follow from FLY_APP_NAME at boot (see entrypoint.sh) and the +# CloudSync image tag from CLOUDSYNC_IMAGE (see docker-compose.cloudsync.yml); +# override either with `fly secrets set`. + +[build] + dockerfile = "Dockerfile" + +[env] + # Envoy replaces Kong as the API gateway. Upstream makes this the default in + # the ~Aug 2026 release, after which the override becomes a no-op shim. + COMPOSE_FILE = "docker-compose.yml:docker-compose.envoy.yml:docker-compose.cloudsync.yml" + SITE_URL = "http://localhost:3000" + POOLER_TENANT_ID = "cloudsync" + STUDIO_DEFAULT_ORGANIZATION = "CloudSync" + STUDIO_DEFAULT_PROJECT = "Supabase" + +[[mounts]] + source = "supabase_data" + destination = "/data" + initial_size = "50gb" + +[[vm]] + size = "shared-cpu-4x" + memory = "4096" + +# API gateway. Gives Studio and the REST/Auth/Storage APIs HTTPS on .fly.dev. +[http_service] + internal_port = 8000 + force_https = true + auto_stop_machines = "off" + auto_start_machines = false + min_machines_running = 1 + +# Postgres (container port 5433) is intentionally NOT published. Same-org apps +# reach it at .internal:5433; laptops use `fly proxy 5433`. To expose it +# publicly, add a [[services]] block with internal_port = 5433 — note that a +# shared IPv4 serves 80/443 only and Postgres cannot use Fly's TLS handler, so +# public IPv4 access needs a dedicated IPv4 (`fly ips allocate-v4`). diff --git a/docker/README.md b/docker/README.md index 2aaff4b..d9e1fa4 100644 --- a/docker/README.md +++ b/docker/README.md @@ -112,6 +112,15 @@ includes CloudSync, then tag it with the same name so the CLI reuses it. This keeps your local Supabase stack intact (auth, realtime, storage, etc.) while enabling the extension in the CLI-managed Postgres container. +> **Note on Ubuntu vs Alpine bases.** Newer Supabase Postgres images (roughly +> `17.6.1.084` and later) use an Alpine-based userland instead of the earlier +> Ubuntu one. The published release image (`Dockerfile.supabase.release`, used +> for `sqlitecloud/sqlite-sync-supabase:*`) detects the userland automatically +> and works on both. The local build-from-source flow below +> (`make postgres-supabase-build`, `Dockerfile.supabase`) currently assumes an +> Ubuntu base; if your CLI stack pulls a newer Alpine image, prefer the +> published `:17-alpine` release image. + ### Prerequisites - Supabase CLI installed (`supabase start` works) diff --git a/docker/postgresql/Dockerfile.supabase.release b/docker/postgresql/Dockerfile.supabase.release index 083e6c7..6e56b43 100644 --- a/docker/postgresql/Dockerfile.supabase.release +++ b/docker/postgresql/Dockerfile.supabase.release @@ -1,14 +1,21 @@ # Supabase PostgreSQL with pre-compiled CloudSync (sqlite-sync) extension # +# Works with both Supabase base image families: +# - Ubuntu/glibc images (e.g. PG15 15.8.1.135, older PG17 17.6.1.071) +# - Alpine-userland images (e.g. PG17 17.6.1.084+); still a glibc Nix postgres +# Both run a Nix-built, glibc-linked PostgreSQL, so the SAME glibc extension +# artifact is installed in either case. Only the build-time userland tooling +# differs (apt-get vs apk), which this file detects at runtime. +# # Usage: # docker build \ -# --build-arg SUPABASE_POSTGRES_TAG=15.8.1.085 \ -# --build-arg CLOUDSYNC_VERSION=1.0.6 \ +# --build-arg SUPABASE_POSTGRES_TAG=17.6.1.151 \ +# --build-arg CLOUDSYNC_VERSION= \ # -f docker/postgresql/Dockerfile.supabase.release \ # -t my-cloudsync-supabase-postgres . # # Or pull the pre-built image from Docker Hub: -# docker pull sqlitecloud/sqlite-sync-supabase:15 +# docker pull sqlitecloud/sqlite-sync-supabase:17 # ARG SUPABASE_POSTGRES_TAG=17.6.1.071 @@ -17,60 +24,97 @@ FROM public.ecr.aws/supabase/postgres:${SUPABASE_POSTGRES_TAG} ARG CLOUDSYNC_VERSION ARG TARGETARCH -ENV CLOUDSYNC_PG_CONFIG=/root/.nix-profile/bin/pg_config +# pg_config is resolved dynamically at build time (its path differs between the +# Ubuntu and Alpine Supabase bases). Pass --build-arg CLOUDSYNC_PG_CONFIG= +# to override. Kept an ARG so it does not leak into the published image. +ARG CLOUDSYNC_PG_CONFIG="" # CI beta builds stage pre-built artifacts here; when the tarball for this arch # is present it is used instead of the GitHub release download COPY docker/postgresql/beta-artifacts/ /tmp/beta-artifacts/ # Download pre-compiled extension and install into Supabase's Nix layout -RUN case "${TARGETARCH}" in \ +RUN set -eu; \ + : "${CLOUDSYNC_VERSION:?required: pass --build-arg CLOUDSYNC_VERSION=}"; \ + case "${TARGETARCH:-}" in \ amd64) ARCH="x86_64" ;; \ arm64) ARCH="arm64" ;; \ - *) echo "Unsupported architecture: ${TARGETARCH}" && exit 1 ;; \ - esac && \ + *) echo "Unsupported architecture: ${TARGETARCH:-}"; exit 1 ;; \ + esac; \ + # Resolve pg_config: the Nix profiles the Supabase bases ship first, PATH last + PG_CONFIG="${CLOUDSYNC_PG_CONFIG:-}"; \ + if [ -z "$PG_CONFIG" ]; then \ + for c in /root/.nix-profile/bin/pg_config /nix/var/nix/profiles/default/bin/pg_config "$(command -v pg_config || true)"; do \ + if [ -x "$c" ]; then PG_CONFIG="$c"; break; fi; \ + done; \ + fi; \ + if [ ! -x "$PG_CONFIG" ]; then echo "Error: pg_config not found${PG_CONFIG:+ at $PG_CONFIG}"; exit 1; fi; \ # Derive PG major version from pg_config - PG_MAJOR=$(${CLOUDSYNC_PG_CONFIG} --version | sed 's/[^0-9]*//' | cut -d. -f1) && \ - apt-get update && apt-get install -y --no-install-recommends curl ca-certificates && \ - ASSET="cloudsync-postgresql${PG_MAJOR}-linux-${ARCH}-${CLOUDSYNC_VERSION}.tar.gz" && \ + PG_MAJOR=$(${PG_CONFIG} --version | sed 's/[^0-9]*//' | cut -d. -f1); \ + ASSET="cloudsync-postgresql${PG_MAJOR}-linux-${ARCH}-${CLOUDSYNC_VERSION}.tar.gz"; \ if [ -f "/tmp/beta-artifacts/${ASSET}" ]; then \ - echo "Using local artifact ${ASSET}" && \ + echo "Using local artifact ${ASSET}"; \ cp "/tmp/beta-artifacts/${ASSET}" /tmp/cloudsync.tar.gz; \ else \ - URL="https://github.com/sqliteai/sqlite-sync/releases/download/${CLOUDSYNC_VERSION}/${ASSET}" && \ - echo "Downloading ${URL}" && \ + # curl is only needed for the release download; install it with whichever + # package manager the userland provides, and remove it afterwards only if + # it was not already part of the base image + HAD_CURL=1; \ + command -v curl >/dev/null 2>&1 || HAD_CURL=0; \ + if command -v apk >/dev/null 2>&1; then \ + PKGMGR=apk; \ + apk add --no-cache curl ca-certificates; \ + elif command -v apt-get >/dev/null 2>&1; then \ + PKGMGR=apt; \ + apt-get update; \ + apt-get install -y --no-install-recommends curl ca-certificates; \ + else \ + echo "No supported package manager (apk/apt-get) found"; exit 1; \ + fi; \ + URL="https://github.com/sqliteai/sqlite-sync/releases/download/${CLOUDSYNC_VERSION}/${ASSET}"; \ + echo "Downloading ${URL}"; \ curl -fSL "${URL}" -o /tmp/cloudsync.tar.gz; \ - fi && \ - mkdir -p /tmp/cloudsync && \ - tar -xzf /tmp/cloudsync.tar.gz -C /tmp/cloudsync && \ - # Resolve Supabase's Nix library path - PKGLIBDIR="$(${CLOUDSYNC_PG_CONFIG} --pkglibdir)" && \ - NIX_PGLIBDIR="$(grep -E '^export NIX_PGLIBDIR' /usr/bin/postgres | sed -E "s/.*'([^']+)'.*/\1/" || true)" && \ - if [ -n "$NIX_PGLIBDIR" ]; then PKGLIBDIR="$NIX_PGLIBDIR"; fi && \ - SHAREDIR_PGCONFIG="$(${CLOUDSYNC_PG_CONFIG} --sharedir)" && \ - SHAREDIR_STD="/usr/share/postgresql" && \ - install -d "$PKGLIBDIR" "$SHAREDIR_PGCONFIG/extension" && \ - install -m 755 /tmp/cloudsync/cloudsync.so "$PKGLIBDIR/" && \ - install -m 644 /tmp/cloudsync/cloudsync--*.sql /tmp/cloudsync/cloudsync.control "$SHAREDIR_PGCONFIG/extension/" && \ - if [ "$SHAREDIR_STD" != "$SHAREDIR_PGCONFIG" ]; then \ - install -d "$SHAREDIR_STD/extension" && \ + if [ "$HAD_CURL" = 0 ]; then \ + if [ "$PKGMGR" = apk ]; then apk del curl; else apt-get purge -y curl; apt-get autoremove -y; fi; \ + fi; \ + if [ "$PKGMGR" = apt ]; then rm -rf /var/lib/apt/lists/*; fi; \ + fi; \ + mkdir -p /tmp/cloudsync; \ + tar -xzf /tmp/cloudsync.tar.gz -C /tmp/cloudsync; \ + # Resolve Supabase's Nix library path (postgres wrapper overrides libdir) + PKGLIBDIR="$(${PG_CONFIG} --pkglibdir)"; \ + NIX_PGLIBDIR="$(grep -E '^export NIX_PGLIBDIR' /usr/bin/postgres | sed -E "s/.*'([^']+)'.*/\1/" || true)"; \ + if [ -n "$NIX_PGLIBDIR" ]; then PKGLIBDIR="$NIX_PGLIBDIR"; fi; \ + SHAREDIR_PGCONFIG="$(${PG_CONFIG} --sharedir)"; \ + SHAREDIR_STD="/usr/share/postgresql"; \ + install -d "$PKGLIBDIR" "$SHAREDIR_PGCONFIG/extension"; \ + install -m 755 /tmp/cloudsync/cloudsync.so "$PKGLIBDIR/"; \ + install -m 644 /tmp/cloudsync/cloudsync--*.sql /tmp/cloudsync/cloudsync.control "$SHAREDIR_PGCONFIG/extension/"; \ + if [ "$SHAREDIR_STD" != "$SHAREDIR_PGCONFIG" ] && [ -d "$SHAREDIR_STD" ]; then \ + install -d "$SHAREDIR_STD/extension"; \ install -m 644 /tmp/cloudsync/cloudsync--*.sql /tmp/cloudsync/cloudsync.control "$SHAREDIR_STD/extension/"; \ - fi && \ - rm -rf /tmp/cloudsync /tmp/cloudsync.tar.gz /tmp/beta-artifacts && \ - apt-get purge -y curl && apt-get autoremove -y && rm -rf /var/lib/apt/lists/* + fi; \ + rm -rf /tmp/cloudsync /tmp/cloudsync.tar.gz /tmp/beta-artifacts # Verify installation -RUN NIX_PGLIBDIR="$(grep -E '^export NIX_PGLIBDIR' /usr/bin/postgres | sed -E "s/.*'([^']+)'.*/\1/" || true)" && \ - echo "Verifying CloudSync extension installation..." && \ +RUN set -eu; \ + PG_CONFIG="${CLOUDSYNC_PG_CONFIG:-}"; \ + if [ -z "$PG_CONFIG" ]; then \ + for c in /root/.nix-profile/bin/pg_config /nix/var/nix/profiles/default/bin/pg_config "$(command -v pg_config || true)"; do \ + if [ -x "$c" ]; then PG_CONFIG="$c"; break; fi; \ + done; \ + fi; \ + NIX_PGLIBDIR="$(grep -E '^export NIX_PGLIBDIR' /usr/bin/postgres | sed -E "s/.*'([^']+)'.*/\1/" || true)"; \ + echo "Verifying CloudSync extension installation..."; \ if [ -n "$NIX_PGLIBDIR" ]; then \ ls -la "$NIX_PGLIBDIR/cloudsync.so"; \ else \ - ls -la "$(${CLOUDSYNC_PG_CONFIG} --pkglibdir)/cloudsync.so"; \ - fi && \ - ls -la "$(${CLOUDSYNC_PG_CONFIG} --sharedir)/extension/cloudsync"* && \ + ls -la "$(${PG_CONFIG} --pkglibdir)/cloudsync.so"; \ + fi; \ + ls -la "$(${PG_CONFIG} --sharedir)/extension/cloudsync"*; \ if [ -d "/usr/share/postgresql/extension" ]; then \ ls -la /usr/share/postgresql/extension/cloudsync*; \ - fi && \ + fi; \ echo "CloudSync extension installed successfully" EXPOSE 5432 diff --git a/docs/internal/postgres-flyio.md b/docs/internal/postgres-flyio.md index 080daad..c9e085c 100644 --- a/docs/internal/postgres-flyio.md +++ b/docs/internal/postgres-flyio.md @@ -869,6 +869,15 @@ cd /data put /absolute/path/to/cloudsync-postgresql17-linux-x86_64.zip cloudsync-postgresql17-linux-x86_64.zip ``` +If the upload fails because `cloudsync-postgresql17-linux-x86_64.zip` already exists on the VM, exit the SFTP prompt and remove it from a normal SSH shell first: + +```bash +fly ssh console --app +rm -f /data/cloudsync-postgresql17-linux-x86_64.zip +``` + +Then start `fly ssh sftp shell` again and rerun `put`. The SFTP prompt only supports a small command set such as `cd`, `ls`, `get`, `put`, and `chmod`; it does not support `rm`. + 3. SSH back into the VM, extract the package, and verify the files: ```bash diff --git a/docs/internal/supabase-flyio-fly-deploy.md b/docs/internal/supabase-flyio-fly-deploy.md new file mode 100644 index 0000000..14ee49a --- /dev/null +++ b/docs/internal/supabase-flyio-fly-deploy.md @@ -0,0 +1,437 @@ +# Self-Hosted Supabase on Fly.io — `fly deploy` Strategy + +How to run a self-hosted Supabase stack with the CloudSync Postgres extension on +Fly.io. Everything lives in `deploy/supabase-flyio/` and is deployed with a single +`fly deploy`. No SSH, no `apt-get`, no editing files on the VM. + +Nothing in `deploy/supabase-flyio/` names a specific deployment: the app name, +region and org are command-line arguments, and the public URLs are derived from +them at boot. The examples below use these shell variables: + +```bash +FLY_APP= +FLY_REGION= +FLY_ORG= +``` + +Hardware for a reference deployment is `shared-cpu-4x` / 4 GB / 50 GB volume, set +in `fly.toml`. + +--- + +## Why not Fly's native Docker Compose support + +Fly can turn a compose file into containers on one Machine +(`[build.compose]` in `fly.toml`, +[docs](https://fly.io/docs/machines/guides-examples/multi-container-machines/)). +It cannot run the Supabase compose file. The converter is +`internal/containerconfig/compose.go` in `superfly/flyctl`, and it: + +| Limitation | Consequence for Supabase | +|---|---| +| Plain `yaml.Unmarshal`, no `${VAR}` interpolation and no `env_file` | Every service would receive the literal string `${POSTGRES_PASSWORD}` | +| `volumes:` entries are `os.ReadFile(hostPath)` → injected as base64 container files | File mounts (`kong.yml`, `*.sql`, `pooler.exs`) work; directory mounts (`volumes/db/data`, `volumes/storage`, `volumes/functions`) are dropped with a warning | +| Never emits `containers[].mounts` | No way to attach a Fly Volume to a container: PGDATA lands on the ephemeral overlay and is wiped on restart/redeploy | +| Named volumes ignored | `db-config:/etc/postgresql-custom` (pgsodium key) is dropped | +| One buildable service; secrets are machine-global; one container gets ingress | Manageable, but on top of the above | + +So we keep real Docker Compose and put dockerd inside the Machine instead. + +--- + +## How it works + +One Fly Machine runs a Docker-in-Docker image. The image contains dockerd, the +compose plugin and a pinned copy of the upstream Supabase `docker/` directory. +A Fly Volume at `/data` holds the Docker data root, PGDATA, storage and `.env`, +so state survives restarts and redeploys. + +``` +deploy/supabase-flyio/ +├── Dockerfile # docker:28-dind + fuse-overlayfs + pinned supabase/docker tree +├── entrypoint.sh # dockerd → compose tree → .env bootstrap → compose up → post-init +├── docker-compose.cloudsync.yml # our overrides, layered via COMPOSE_FILE +├── cloudsync.sql # CREATE EXTENSION IF NOT EXISTS cloudsync +└── fly.toml # hardware, volume, env, HTTPS service +``` + +Notable decisions: + +- **`fuse-overlayfs`**: Docker's `overlay2` fails inside a Fly Machine + (`failed to convert whiteout file: operation not permitted`). `vfs` works but + costs ~35 GB and 30–60 min for a full pull. +- **Docker data root on `/data/docker`**: images are pulled once, not on every boot. +- **Compose config re-copied from the image on every boot**, `.env` and + `volumes/` are not — config is immutable and versioned, state is persistent. +- **Overrides layered via `COMPOSE_FILE`** (upstream's own mechanism) instead of + `sed`-ing `docker-compose.yml`, so re-pinning upstream needs no merge work. +- **dockerd runs with IPv6 enabled** so published ports also bind `[::]`. Fly's + private network (`.internal`) is IPv6-only, and Docker publishes IPv4-only + by default — without this, server-to-server access over `.internal` fails. +- **`docker compose logs -f` is the foreground process**, so `fly logs` shows all + services. If it exits, the Machine restarts and the bootstrap re-runs. +- **A failed `compose up` does not kill the Machine.** It used to: with `set -e`, + one service failing its dependency condition exited the entrypoint, Fly + restarted, it failed again, and after 10 rounds the Machine was stopped — taking + the healthy services with it and leaving nothing to inspect. Now the failure is + logged and the boot continues to the log stream. +- **Containers of services dropped from the config are removed before `up`.** + `--remove-orphans` does not cover a service that still exists but sits in a + disabled profile, so its container keeps running *and keeps its published + ports*. That is exactly what happens when swapping the gateway: the old + `supabase-kong` container held port 8000 and Envoy could not bind. + +### API gateway: Envoy + +`COMPOSE_FILE` layers `docker-compose.envoy.yml`, which parks Kong in a disabled +profile and runs `envoyproxy/envoy` as `api-gw` / `supabase-envoy` on the same +host port 8000, keeping the network aliases `kong` and `envoy` so other services' +internal URLs still resolve. Upstream makes Envoy the default in the ~August 2026 +release ([discussion 48048](https://github.com/orgs/supabase/discussions/48048)); +after that the override becomes a no-op shim and Kong moves to +`docker-compose.kong.yml`. + +The Envoy default has no `:8443` HTTPS listener, which does not matter here — +Fly terminates TLS at the edge and only plain 8000 is ever exposed. + +Envoy is stricter than Kong about credentials. `/rest/v1/` (the PostgREST OpenAPI +root) is service-role-only and answers `RBAC: access denied` to an anon key; that +is intended hardening, not a broken route. + +### Overrides applied to upstream compose + +```yaml +services: + db: + image: ${CLOUDSYNC_IMAGE:-sqlitecloud/sqlite-sync-supabase:17-alpine} + volumes: + - ./cloudsync.sql:/docker-entrypoint-initdb.d/init-scripts/100-cloudsync.sql:Z + ports: + - "5433:5432" # direct Postgres; 5432 belongs to Supavisor + supavisor: + entrypoint: ["/usr/bin/tini", "-s", "-g", "--"] # skips /app/limits.sh +``` + +The `100-` prefix keeps the CloudSync init script after Supabase's own scripts +(97–99). Init scripts only run on an empty PGDATA, so `entrypoint.sh` also runs +`CREATE EXTENSION IF NOT EXISTS cloudsync;` on every boot. + +--- + +## Deploying + +```bash +fly apps create "$FLY_APP" --org "$FLY_ORG" +cd deploy/supabase-flyio +fly deploy . --config fly.toml --app "$FLY_APP" --primary-region "$FLY_REGION" +``` + +That is the whole setup. The volume is created from `[[mounts]] initial_size`, +IPs are allocated automatically, and the first boot generates all secrets. + +`fly.toml` deliberately has no `app` or `primary_region` key, so the same file +works for any deployment and no environment-specific identifier is committed. + +First boot pulls ~11 service images and takes roughly 5–10 minutes. Follow it with +`fly logs -a "$FLY_APP"`. + +To run a pre-release CloudSync image instead of the tag baked into the compose +override: + +```bash +fly secrets set -a "$FLY_APP" \ + CLOUDSYNC_IMAGE=sqlitecloud/sqlite-sync-supabase:17-alpine-beta- +``` + +### Secrets + +`entrypoint.sh` bootstraps `.env` on first boot only, from `.env.example` plus +upstream's own generators: + +``` +sh ./utils/generate-keys.sh --update-env # JWT_SECRET, ANON_KEY, SERVICE_ROLE_KEY, … +sh ./utils/add-new-auth-keys.sh --update-env # ES256 JWT_KEYS/JWT_JWKS, publishable/secret keys +``` + +plus a random `POSTGRES_PASSWORD` and `DASHBOARD_PASSWORD`. Read them back with: + +```bash +fly ssh console -a "$FLY_APP" -C \ + "grep -E '^(POSTGRES_PASSWORD|DASHBOARD_USERNAME|DASHBOARD_PASSWORD|ANON_KEY|SERVICE_ROLE_KEY|JWT_SECRET)=' /data/supabase-docker/.env" +``` + +To pin a value yourself, set it as a Fly secret — compose interpolation prefers +the process environment over `.env`, and Fly secrets are process environment: + +```bash +fly secrets set -a "$FLY_APP" POSTGRES_PASSWORD=... +``` + +> Changing `POSTGRES_PASSWORD` *after* the database has been initialized leaves +> the existing roles on the old password, and services then fail with +> `FATAL: password authentication failed for user "authenticator"`. Realign them: +> +> ```bash +> cd /data/supabase-docker +> for role in authenticator supabase_auth_admin supabase_storage_admin supabase_admin; do +> docker compose exec -T db psql -U postgres \ +> -c "ALTER USER $role WITH PASSWORD 'NEW_PASSWORD';" +> done +> docker compose restart +> ``` +> +> Upstream also ships `sh ./utils/db-passwd.sh` for a guided rotation, followed by +> `docker compose up -d --force-recreate`. + +--- + +## Accessing the stack + +| What | How | +|---|---| +| Studio / API gateway | `https://$FLY_APP.fly.dev` (basic auth: `DASHBOARD_USERNAME` / `DASHBOARD_PASSWORD`) | +| REST / Auth / Storage / Realtime | `https://$FLY_APP.fly.dev/{rest,auth,storage,realtime}/v1/` | +| Postgres, direct, from another Fly app | `postgres://postgres:@$FLY_APP.internal:5433/postgres` | +| Postgres, direct, from a laptop | `fly proxy 5433 -a "$FLY_APP"` then connect to `localhost:5433` | +| Postgres via Supavisor pooler | machine port `5432` (session) / `6543` (transaction), username `postgres.$POOLER_TENANT_ID` | +| Shell | `fly ssh console -a "$FLY_APP"` then `cd /data/supabase-docker` | + +Register with the CloudSync server using the direct (non-pooled) connection +string on port 5433, `"flavor": "supabase"`. + +To keep everything private instead, drop `[http_service]` from `fly.toml`, +release the public IPs, and use `fly proxy 8000 -a `. + +--- + +## Maintenance + +| Task | Command | +|---|---| +| Update the CloudSync image | released tag: redeploy (compose pulls `:17-alpine`); beta: `fly secrets set CLOUDSYNC_IMAGE=…` | +| Update Supabase services | bump `SUPABASE_REF` in the `Dockerfile`, `fly deploy` | +| Restart the stack | `fly machine restart -a ` — the entrypoint re-runs everything | +| Inspect containers | `fly ssh console -a -C "docker ps"` | +| Resize hardware | `fly scale vm shared-cpu-8x --vm-memory 8192 -a ` | +| Grow the volume | `fly volumes extend --size 80 -a ` | +| Destroy | `fly apps destroy ` (removes the volume too) | + +Redeploys recreate the Machine but keep `/data`, so the Docker image cache and +the database survive; only the ~135 MB app image is re-pulled. + +--- + +## Registering the database with the CloudSync server + +The CloudSync server needs a Postgres connection string it can reach. If it runs +in the same Fly org, use the private network directly — no public exposure needed: + +```bash +export CLOUDSYNC_URL="" +export ORG_API_KEY="" +export CONNECTION_STRING="postgres://postgres:@$FLY_APP.internal:5433/postgres" + +curl "$CLOUDSYNC_URL/healthz" # {"status":"ok"} +``` + +If the server runs outside Fly, tunnel instead: `fly proxy 5433 -a "$FLY_APP"` and +use `@localhost:5433`. + +A managed database belongs to a workspace, and an org-wide key must name one. +`PUT` is idempotent — it returns the existing workspace if the slug is taken: + +```bash +curl --request PUT "$CLOUDSYNC_URL/v1/workspaces/" \ + --header "Authorization: Bearer $ORG_API_KEY" \ + --header "Content-Type: application/json" \ + --data '{"name": "Supabase PG17 Test"}' # → workspaceId +``` + +Register the database. Because GoTrue here is HS256-only (see [Client +token](#client-token)), pass `jwtSecret` and the issuer/audience so CloudSync +validates tokens with the shared secret instead of JWKS: + +```bash +curl --request POST "$CLOUDSYNC_URL/v1/databases" \ + --header "Authorization: Bearer $ORG_API_KEY" \ + --header "Content-Type: application/json" \ + --data '{ + "label": "Supabase Fly.io Test", + "connectionString": "'"$CONNECTION_STRING"'", + "provider": "postgres", + "flavor": "supabase", + "projectId": "'"$FLY_APP"'", + "databaseName": "postgres", + "workspaceId": "", + "jwtSecret": "", + "jwtAllowedIssuers": ["https://'"$FLY_APP"'.fly.dev/auth/v1"], + "jwtExpectedAudiences": ["authenticated"] + }' + +export MANAGED_DATABASE_ID="" +``` + +The response should show `"auth": {"mode": "jwt_hmac", …}`. Always register the +**direct** port 5433, not the Supavisor pooler. There is no `/verify` endpoint — +the `cloudsync/tables` call below doubles as the connectivity check, since it only +answers once the server has reached the database. + +Create a table and enable sync on it: + +```bash +fly ssh console -a "$FLY_APP" +cd /data/supabase-docker && docker compose exec db psql -U postgres +``` + +```sql +CREATE TABLE IF NOT EXISTS todos ( + id TEXT PRIMARY KEY DEFAULT cloudsync_uuid(), + title TEXT NOT NULL DEFAULT '', + done BOOLEAN DEFAULT false +); +SELECT cloudsync_init('todos'); +``` + +```bash +curl --request POST "$CLOUDSYNC_URL/v1/databases/$MANAGED_DATABASE_ID/cloudsync/enable" \ + --header "Authorization: Bearer $ORG_API_KEY" \ + --header "Content-Type: application/json" \ + --data '{"tables":["todos"]}' + +curl --request GET "$CLOUDSYNC_URL/v1/databases/$MANAGED_DATABASE_ID/cloudsync/tables" \ + --header "Authorization: Bearer $ORG_API_KEY" # todos should show "enabled": true +``` + +### Client token + +For a `supabase`-flavored database the client authenticates with a GoTrue JWT, not +an org key. Create the user once, then take an `access_token` per session: + +```bash +ANON_KEY=$(fly ssh console -a "$FLY_APP" -C \ + "grep ^ANON_KEY= /data/supabase-docker/.env" | tail -1 | tr -d '\r' | cut -d= -f2-) + +curl -X POST "https://$FLY_APP.fly.dev/auth/v1/signup" \ + -H "apikey: $ANON_KEY" -H "Content-Type: application/json" \ + -d '{"email":"sync-test@example.com","password":"..."}' + +curl -X POST "https://$FLY_APP.fly.dev/auth/v1/token?grant_type=password" \ + -H "apikey: $ANON_KEY" -H "Content-Type: application/json" \ + -d '{"email":"sync-test@example.com","password":"..."}' # → access_token +``` + +Tokens are HS256, signed with `JWT_SECRET` from `.env`, audience `authenticated`, +issuer `https://$FLY_APP.fly.dev/auth/v1`. Read the issuer from the service, not +from `.env`: the entrypoint derives `API_EXTERNAL_URL` from the app name and +exports it, so process env wins and the `.env` line still reads +`http://localhost:8000/auth/v1`. Confirm with +`docker compose exec auth printenv GOTRUE_JWT_ISSUER`. Validate one with +`GET /auth/v1/user` + `Authorization: Bearer `. Note that +`/auth/v1/.well-known/jwks.json` returns `{"keys":[]}`: upstream ships +`GOTRUE_JWT_KEYS` commented out, so GoTrue stays HS256-only even though `.env` +carries the ES256 material (PostgREST does use it, via `PGRST_JWT_SECRET`). + +### Roundtrip + +```sql +-- SQLite client (Homebrew sqlite3; the system one cannot load extensions) +.load dist/cloudsync.dylib +CREATE TABLE todos ( + id TEXT PRIMARY KEY DEFAULT (cloudsync_uuid()), + title TEXT NOT NULL DEFAULT '', + done INTEGER DEFAULT 0 +); +SELECT cloudsync_init('todos'); +SELECT cloudsync_network_init_custom('', ''); +SELECT cloudsync_network_set_token(''); + +INSERT INTO todos (id, title) VALUES (cloudsync_uuid(), 'from SQLite'); +SELECT cloudsync_network_send_changes(); +SELECT cloudsync_terminate(); +``` + +Then check the row landed in Postgres, insert one there, and pull it back with +`SELECT cloudsync_network_receive_changes();` on the client — allow ~10 s, the +server picks up Postgres-side changes on its own cadence. + +Three things that will bite otherwise: + +- **Declare the SQLite column `INTEGER`, not `BOOLEAN`, for a Postgres `boolean`.** + The schema hash normalizes types per side: PostgreSQL maps `boolean` → `integer`, + while SQLite maps a `BOOLEAN` declaration → `text`. The hashes then differ and the + server rejects the payload with + `Cannot apply the received payload because the schema hash is unknown `. +- **`cloudsync_uuid()` in a `DEFAULT` clause fails on insert** with + `unsafe use of cloudsync_uuid()` unless `PRAGMA trusted_schema=ON`. Pass the id + explicitly instead: `INSERT INTO todos (id, …) VALUES (cloudsync_uuid(), …)`. +- **`cloudsync_network_init` targets the default server.** For any other CloudSync + server use `cloudsync_network_init_custom('', '')`. Neither this nor + `cloudsync_network_set_token` is persisted — both must run in every session. + +--- + +## Building a custom CloudSync Postgres image + +CI publishes a beta tag for every branch push (see the publish matrix in +`.github/workflows/main.yml`), which is usually all you need — point the machine at +it with `CLOUDSYNC_IMAGE`. To build one locally instead: + +```bash +git submodule update --init --recursive # else: fractional_indexing.h: No such file or directory + +docker build --platform linux/amd64 \ + --build-arg SUPABASE_POSTGRES_TAG=17.6.1.151 \ + -f docker/postgresql/Dockerfile.supabase \ + -t /: . + +docker push /: +fly secrets set -a "$FLY_APP" CLOUDSYNC_IMAGE=/: +``` + +`--platform linux/amd64` matters: Fly Machines are x86_64, and an Apple Silicon +build produces an ARM image that will not start. The image must be pullable +without credentials, or `docker login` has to run on the Machine. + +--- + +## Troubleshooting + +| Problem | Fix | +|---|---| +| `Bind for 0.0.0.0:8000 failed: port is already allocated` | A container from a previous compose config still holds the port. The entrypoint now removes containers whose service left the config; otherwise `docker rm -f `. | +| Machine stopped, `machine has reached its max restart count of 10` | Something failed during boot and crash-looped. Read `fly logs`, fix, then `fly machine start `. | +| `password authentication failed for user "authenticator"` | Roles hold a different password than `.env` — see the `ALTER USER` block under [Secrets](#secrets). | +| `cloudsync_version()` does not exist | Init scripts only run on an empty PGDATA. `docker compose exec -T db psql -U postgres -c "CREATE EXTENSION IF NOT EXISTS cloudsync;"` (the entrypoint does this on every boot). | +| Auth restarts, `must be owner of function uid (SQLSTATE 42501)` | GoTrue migrates as `supabase_auth_admin` and must own `auth.uid()`/`role()`/`email()`. Automated in `entrypoint.sh`; rerun by deleting `/data/.cloudsync-post-init-done` and restarting. | +| Supavisor crashes on `ulimit: open files: cannot modify limit` | The Fly kernel refuses `/app/limits.sh`. Handled by the entrypoint override in `docker-compose.cloudsync.yml`. | +| `RBAC: access denied` on `/rest/v1/` | Envoy restricts the PostgREST OpenAPI root to the service-role key. Use a table path, or the service-role key. | +| `.internal` unreachable *from inside a container* | Containers have no route to Fly's 6PN. Use `db:5432` (compose network), `127.0.0.1:5432` (inside `supabase-db`), or the bridge gateway on `:5433`. | +| Services unhealthy right after boot | Give it ~2 minutes, then `docker compose logs `. Studio and Envoy come last. | +| `no space left on device` during a pull | `fly volumes extend --size 80 -a "$FLY_APP"`. | +| `cannot stop container … did not receive an exit event` | Zombie container: `kill -9 $(pidof dockerd) $(pidof containerd)`, `rm -rf /data/docker/containers/*`, then restart the Machine. | +| Cannot pull the CloudSync image | The tag must be public, or run `docker login` on the Machine. | + +--- + +## Image / version notes + +The Alpine CloudSync images are built from `supabase/postgres:17.6.1.151` (see the +publish matrix in `.github/workflows/main.yml`), while the pinned upstream compose +expects `17.6.1.136`. Same PG 17 line, newer patch base — fine, but re-check after +bumping either side. + +The pinned compose runs 11 services: `db`, `supavisor`, `auth`, `rest`, `realtime`, +`storage`, `imgproxy`, `meta`, `functions`, `studio` and the gateway. Upstream has +dropped `analytics` (Logflare) and `vector`, so nothing here needs the old +"disable Logflare" workaround. `db` no longer publishes 5432 either — Supavisor +owns it, which is why the override maps direct Postgres to 5433. + +Verified on the first deployment of this setup (`shared-cpu-4x` / 4 GB / 50 GB, +beta image `…:17-alpine-beta-`): all 11 services +healthy on first boot, `PostgreSQL 17.6`, `cloudsync` extension `1.1` / +`cloudsync_version()` = `1.1.2`, `cloudsync_init()` + insert produced a row in +`cloudsync_changes`, the gateway reachable over HTTPS (Envoy: `/auth/v1/health` and +`/storage/v1/version` 200 with the anon key, `/rest/v1/` 200 with the service-role +key), and ports 5432/5433/8000 published +on both `0.0.0.0` and `[::]` (so `.internal` works). First boot took ~9 +minutes, almost all of it image pulls. diff --git a/docs/internal/supabase-flyio.md b/docs/internal/supabase-flyio.md deleted file mode 100644 index 762d0ab..0000000 --- a/docs/internal/supabase-flyio.md +++ /dev/null @@ -1,1001 +0,0 @@ -# Self-Hosting Supabase on Fly.io with CloudSync Extension - -This guide walks you through deploying a full self-hosted Supabase stack on a Fly.io VM, with the CloudSync PostgreSQL extension pre-installed. By the end you will have: - -- A Fly.io VM running all 13 Supabase services via Docker Compose -- PostgreSQL with the CloudSync CRDT extension baked in -- Supabase Studio dashboard accessible over HTTPS -- A custom Postgres image published to Docker Hub - -## Prerequisites - -Install these on your **local machine** before starting: - -| Tool | Purpose | Install | -|------|---------|---------| -| [Docker Desktop](https://www.docker.com/products/docker-desktop/) | Build the custom Postgres image | See [Installing Docker Desktop](#installing-docker-desktop) below | -| [Fly CLI (`flyctl`)](https://fly.io/docs/flyctl/install/) | Provision and manage Fly.io machines | `brew install flyctl` (macOS) or `curl -L https://fly.io/install.sh \| sh` | -| [Git](https://git-scm.com/) | Clone repositories | `brew install git` (macOS) | -| [Docker Hub](https://hub.docker.com/) account | Host your custom Postgres image | Free signup at hub.docker.com | - -You also need a [Fly.io account](https://fly.io/app/sign-up). A credit card is required even for free tier. - -### Installing Docker Desktop - -Docker Desktop is the application that lets you build and run container images on your Mac. - -1. **Download** from https://www.docker.com/products/docker-desktop/ — pick **Apple chip** (M1/M2/M3/M4) or **Intel chip** depending on your Mac. - - > Not sure which you have? Click the Apple menu () → **About This Mac**. It will say either "Apple M1/M2/M3/M4" or "Intel". - -2. **Install**: Open the downloaded `.dmg` file and drag Docker into your Applications folder. - -3. **Launch**: Open Docker from Applications (or Spotlight: Cmd+Space → type "Docker"). It will ask for your password to install system components — that's normal. - -4. **Wait**: A whale icon appears in your menu bar. Wait until it says "Docker Desktop is running" (the whale stops animating). - -5. **Verify** in Terminal: - ```bash - docker --version - # Should output: Docker version 27.x.x or similar - ``` - -### Setting up Docker Hub - -Docker Hub is a free cloud registry where you'll upload your custom Postgres image so the Fly.io server can download it. - -1. **Sign up** at https://hub.docker.com/signup — pick a username. This becomes your image prefix (e.g., `myusername/supabase-postgres-cloudsync`). - -2. **Log in from Terminal**: - ```bash - docker login - ``` - Enter your Docker Hub username and password. You should see "Login Succeeded". - -### Fly.io VM requirements - -Supabase runs 13 services simultaneously (Postgres, Auth, PostgREST, Realtime, Studio, Kong, Storage, etc.), which is why it needs more resources than a typical single app. - -| Resource | Minimum | Recommended | -|----------|---------|-------------| -| RAM | 4 GB | 8 GB+ | -| CPU | 2 cores | 4 cores | -| Disk | 50 GB SSD | 80 GB+ | - ---- - -## Step 1: Initialize git submodules - -The CloudSync extension depends on the [fractional-indexing](https://github.com/sqliteai/fractional-indexing) library, which is included as a git submodule. If you haven't done this already, initialize it: - -```bash -cd /path/to/sqlite-sync-dev -git submodule update --init --recursive -``` - -Without this, the build will fail with `fractional_indexing.h: No such file or directory`. - ---- - -## Step 2: Build the custom Supabase Postgres image - -> **Important — match the Postgres version!** Check which Postgres version the Supabase docker-compose uses by looking at the `db` service `image` tag in `docker-compose.yml` (e.g., `supabase/postgres:15.8.1.085` means PG 15). You must build your custom image with the **same tag**. Using the wrong version will cause init script failures. - -The `make postgres-supabase-build` command does the following: - -1. **Pulls the official Supabase Postgres base image** (e.g., `public.ecr.aws/supabase/postgres:15.8.1.085`) — this is Supabase's standard PostgreSQL image that ships with ~30 extensions pre-installed (PostGIS, pgvector, etc.) -2. **Runs a multi-stage Docker build** using `docker/postgresql/Dockerfile.supabase`: - - **Stage 1 (builder)**: Installs C build tools (`gcc`, `make`), copies the CloudSync source code (`src/`, `modules/`), and compiles `cloudsync.so` against Supabase's `pg_config` - - **Stage 2 (runtime)**: Starts from a clean Supabase Postgres image and copies in just three kinds of file: - - `cloudsync.so` — the compiled extension binary - - `cloudsync.control` — tells PostgreSQL the extension's name and default version (generated at build time from `cloudsync.control.in`, with the version read from `src/cloudsync.h`) - - `cloudsync--.sql` — the SQL that defines all CloudSync functions for the current release (e.g. `cloudsync--1.0.16.sql`), plus any `cloudsync----.sql` upgrade scripts shipped under `src/postgresql/migrations/` -3. **Tags the result** with the same name as the base image, so it's a drop-in replacement - -To find the correct tag, clone the Supabase repo and check: - -```bash -grep 'image: supabase/postgres:' supabase/docker/docker-compose.yml -# Example output: image: supabase/postgres:15.8.1.085 -# Use the version after the colon as your SUPABASE_POSTGRES_TAG -``` - -Run from the sqlite-sync-dev repo root: - -```bash -make postgres-supabase-build SUPABASE_POSTGRES_TAG=15.8.1.085 -``` - -Verify the image was built: - -```bash -docker images | grep supabase-postgres-cloudsync -# Should show: /supabase-postgres-cloudsync 15.8.1.085 ... -``` - -Verify CloudSync is installed inside the image: - -```bash -docker run --rm /supabase-postgres-cloudsync:15.8.1.085 \ - find / -name "cloudsync*" -type f 2>/dev/null -# Should list cloudsync.so, cloudsync.control, and cloudsync--.sql -# (plus any cloudsync----.sql upgrade scripts) -# in /nix/store/...-postgresql-and-plugins-15.8/ paths -``` - ---- - -## Step 3: Build for the correct architecture and push to Docker Hub - -The Fly.io VM needs to pull your custom image from a container registry. We use Docker Hub (free, no extra auth needed on the VM). - -> **Important — architecture mismatch**: If you're building on an Apple Silicon Mac (M1/M2/M3/M4), `make postgres-supabase-build` produces an ARM image. Fly.io VMs run x86 (amd64) by default, so the ARM image won't work. You must build for the target architecture explicitly. - -First, pull the base image for amd64 (this ensures Docker has the correct platform variant cached): - -```bash -docker pull --platform linux/amd64 public.ecr.aws/supabase/postgres:15.8.1.085 -``` - -Then build for `linux/amd64` (x86, which is what Fly.io uses): - -```bash -docker build --platform linux/amd64 \ - --build-arg SUPABASE_POSTGRES_TAG=15.8.1.085 \ - -f docker/postgresql/Dockerfile.supabase \ - -t /supabase-postgres-cloudsync:15.8.1.085 \ - . -``` - -Push the image (you must be logged in: `docker login`): - -```bash -docker push /supabase-postgres-cloudsync:15.8.1.085 -``` - -> **Note**: `docker buildx build ... --push` may fail with ECR registry resolution errors. The two-step approach above (build then push) is more reliable. - -> If you're building on an Intel Mac or a Linux x86 machine, `make postgres-supabase-build` already produces an amd64 image, so you can simply tag and push: -> ```bash -> docker tag public.ecr.aws/supabase/postgres:15.8.1.085 \ -> /supabase-postgres-cloudsync:15.8.1.085 -> docker push /supabase-postgres-cloudsync:15.8.1.085 -> ``` - ---- - -## Step 4: Provision a Fly.io VM - -We use a Fly Machine as a plain Linux VM running Docker Compose — not Fly's container orchestration. - -### 4a. Log in to Fly - -```bash -fly auth login -``` - -This opens your browser to authenticate with your Fly.io account. - -### 4b. Create a Fly app - -```bash -fly apps create -``` - -### 4c. Create a persistent volume for data - -```bash -fly volumes create supabase_data --app --region --size 50 -``` - -Pick a [region](https://fly.io/docs/reference/regions/) close to you. You can see all available regions with `fly platform regions`. Common choices: - -| Code | Location | -|------|----------| -| `fra` | Frankfurt, Germany | -| `ams` | Amsterdam, Netherlands | -| `lhr` | London, UK | -| `ord` | Chicago, US | -| `iad` | Virginia, US | -| `sin` | Singapore | - -When prompted "Do you still want to use the volumes feature?", type `y` — the warning about multiple volumes is for high-availability production setups; a single volume is fine for testing. - -### 4d. Create a Fly Machine - -```bash -fly machine run ubuntu:24.04 \ - --app \ - --region \ - --vm-size shared-cpu-4x \ - --vm-memory 4096 \ - --volume supabase_data:/data \ - --name supabase-vm \ - -- sleep inf -``` - -The `-- sleep inf` at the end is important — it tells the VM to run an infinite sleep process so it stays alive. Without it, the Ubuntu container exits immediately and the machine stops. - -This creates an Ubuntu 24.04 VM with 4 CPU cores, 4 GB RAM, and your 50 GB volume mounted at `/data`. - -The VM size (`shared-cpu-4x` + 4096 MB) meets Supabase's minimum requirements. For a test/dev deployment this is fine. You can resize later with `fly machine update` if needed. - -### 4e. Allocate a public IP - -```bash -fly ips allocate-v4 --shared --app -fly ips allocate-v6 --app -``` - -Note the IPv4 address — you'll need it for `SUPABASE_PUBLIC_URL`. - ---- - -## Step 5: Set up Docker and Supabase on the VM - -### 5a. SSH into the machine - -```bash -fly ssh console --app -``` - -### 5b. Install Docker Engine - -```bash -apt-get update -apt-get install -y ca-certificates curl gnupg - -install -m 0755 -d /etc/apt/keyrings -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg -chmod a+r /etc/apt/keyrings/docker.gpg - -echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] \ - https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo $VERSION_CODENAME) stable" \ - > /etc/apt/sources.list.d/docker.list - -apt-get update -apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin - -# Verify -docker --version -docker compose version -``` - -### 5c. Configure Docker storage driver - -Docker's default `overlayfs` storage driver doesn't work inside a Fly VM (you'll get "failed to convert whiteout file: operation not permitted" errors). Use `fuse-overlayfs` instead — it works in unprivileged environments like Fly VMs and is much faster and more space-efficient than the `vfs` fallback. - -Install fuse-overlayfs: - -```bash -apt-get install -y fuse-overlayfs -``` - -Configure Docker to use it: - -```bash -mkdir -p /etc/docker -echo '{"storage-driver":"fuse-overlayfs","data-root":"/data/docker"}' > /etc/docker/daemon.json -``` - -> **Why not `vfs`?** The `vfs` driver copies every image layer in full instead of sharing them. This means 13 Supabase images can use 35GB+ of disk (vs ~5-8GB with fuse-overlayfs), and image pulls/extraction are extremely slow (30-60 minutes vs a few minutes). Avoid `vfs` unless fuse-overlayfs doesn't work. - -### 5d. Start the Docker daemon - -The Fly VM doesn't auto-start Docker. You need to start it manually: - -```bash -dockerd & -``` - -Wait for the `API listen on /var/run/docker.sock` message before running any Docker commands. This message means Docker is ready. - -> **Note**: If Docker is already running, you'll see a "process is still running" error. That's fine — it means Docker is already available. - -### 5e. Clone and set up Supabase - -```bash -cd /data -git clone --depth 1 https://github.com/supabase/supabase -mkdir -p supabase-docker -cp -rf supabase/docker/* supabase-docker/ -cp supabase/docker/.env.example supabase-docker/.env -cd supabase-docker -``` - ---- - -## Step 6: Configure secrets - -### 6a. Generate keys automatically - -```bash -sh ./utils/generate-keys.sh -``` - -Review the output. The script updates `.env` with generated `JWT_SECRET`, `ANON_KEY`, and `SERVICE_ROLE_KEY`. - -### 6a.1. Get the JWT secret later - -If you need the Supabase Auth JWT secret after setup, read the `JWT_SECRET` value from the same `.env` file used by Docker Compose: - -```bash -cd /data/supabase-docker -grep '^JWT_SECRET=' .env -``` - -That value is the secret GoTrue (Supabase Auth) uses to sign and verify access tokens. - -If you want to confirm what the running auth container sees, check the container environment: - -```bash -docker compose exec auth printenv GOTRUE_JWT_SECRET -``` - -Both commands should return the same value. If they do not, restart the stack after updating `.env`: - -```bash -docker compose up -d -``` - -### 6b. Edit `.env` manually for remaining values - -```bash -# Install a text editor if needed -apt-get install -y nano -nano .env -``` - -Set these values: - -```env -############ -# Required # -############ - -# Database — letters and numbers only, no special characters -POSTGRES_PASSWORD= - -# URLs — replace with your Fly app's public IP or domain -SUPABASE_PUBLIC_URL=http://:8000 -API_EXTERNAL_URL=http://:8000 -SITE_URL=http://localhost:3000 - -# Dashboard login credentials -DASHBOARD_USERNAME=supabase -DASHBOARD_PASSWORD= - -############ -# Secrets # -############ -# These should already be set by generate-keys.sh, but verify they exist: -# JWT_SECRET= -# ANON_KEY= -# SERVICE_ROLE_KEY= - -# Generate the rest if not already set: -# openssl rand -base64 48 → SECRET_KEY_BASE -# openssl rand -hex 16 → VAULT_ENC_KEY (must be exactly 32 chars) -# openssl rand -base64 24 → PG_META_CRYPTO_KEY -# openssl rand -base64 24 → LOGFLARE_PUBLIC_ACCESS_TOKEN -# openssl rand -base64 24 → LOGFLARE_PRIVATE_ACCESS_TOKEN -# openssl rand -hex 16 → S3_PROTOCOL_ACCESS_KEY_ID -# openssl rand -hex 32 → S3_PROTOCOL_ACCESS_KEY_SECRET -# openssl rand -hex 16 → MINIO_ROOT_PASSWORD -``` - ---- - -## Step 7: Swap in the CloudSync Postgres image - -Edit `docker-compose.yml` and find the `db` service (near the top). Replace the `image` line: - -```yaml -services: - db: - # BEFORE: image: supabase/postgres:${POSTGRES_VERSION} - # AFTER: - image: sqlitecloud/sqlite-sync-supabase:15.8.1.085 -``` - -Use `sqlitecloud/sqlite-sync-supabase:15.8.1.085` if you want the published image. If you built and pushed your own image in Step 3 for internal testing, use that exact image path instead. - -### Add the CloudSync init script - -Create the init SQL: - -```bash -cat > volumes/db/cloudsync.sql << 'EOF' -CREATE EXTENSION IF NOT EXISTS cloudsync; -EOF -``` - -Add a volume mount to the `db` service in `docker-compose.yml`: - -```yaml -services: - db: - volumes: - # ... existing volume mounts ... - - ./volumes/db/cloudsync.sql:/docker-entrypoint-initdb.d/init-scripts/100-cloudsync.sql:Z -``` - -The `100-` prefix ensures CloudSync loads after Supabase's own init scripts (numbered 97-99). - -> **Important**: Init scripts only run when the data directory is empty (first start). If you've already started Postgres once and need to add the extension, connect and run `CREATE EXTENSION cloudsync;` manually. - ---- - -## Step 8: Start Supabase - -```bash -cd /data/supabase-docker -docker compose pull -docker compose up -d -``` - -Wait ~1 minute for all services to start, then verify: - -```bash -docker compose ps -``` - -All services should show `Up (healthy)`. If any service is unhealthy: - -```bash -docker compose logs -``` - -### Fix: services fail with "password authentication failed" - -If you see `FATAL: password authentication failed for user "authenticator"` (or `supabase_auth_admin`, `supabase_storage_admin`, `supabase_admin`) in the logs, the database users were created with a different password than what's in `.env`. This happens when `POSTGRES_PASSWORD` was changed after the first start, or when the DB data persists across reinstalls. - -Fix by updating all user passwords to match your `.env`: - -```bash -# Replace YOUR_PASSWORD with the value of POSTGRES_PASSWORD from your .env file -docker compose exec db psql -U postgres -c "ALTER USER authenticator WITH PASSWORD 'YOUR_PASSWORD';" -docker compose exec db psql -U postgres -c "ALTER USER supabase_auth_admin WITH PASSWORD 'YOUR_PASSWORD';" -docker compose exec db psql -U postgres -c "ALTER USER supabase_storage_admin WITH PASSWORD 'YOUR_PASSWORD';" -docker compose exec db psql -U postgres -c "ALTER USER supabase_admin WITH PASSWORD 'YOUR_PASSWORD';" -``` - -Then restart: - -```bash -docker compose restart -``` - -### Fix: analytics (Logflare) keeps crashing - -The analytics service (Logflare) often fails in self-hosted setups due to migration issues. Since it's **optional** (only used for log analytics, not required for CloudSync or core Supabase features), the simplest fix is to disable it. - -In `docker-compose.yml`: - -1. **Remove the `analytics` dependency** from the `studio` service's `depends_on` block. Delete these lines: - ```yaml - analytics: - condition: service_healthy - ``` - If `depends_on:` becomes empty after removing, delete the `depends_on:` line too. - -2. **Comment out the `LOGFLARE_URL`** environment variable in the `studio` service: - ```yaml - # LOGFLARE_URL: http://analytics:4000 - ``` - -3. **Restart**: - ```bash - docker compose stop analytics - docker compose up -d - ``` - -### Fix: CloudSync extension not found - -If `SELECT cloudsync_version();` returns "function does not exist", the init script didn't run (it only runs on first boot when the data directory is empty). Create the extension manually: - -```bash -docker compose exec db psql -U postgres -c "CREATE EXTENSION IF NOT EXISTS cloudsync;" -``` - -### Fix: Auth service crashes with "must be owner of function uid" - -When the auth (GoTrue) service fails to start and logs show errors like: - -``` -error executing migrations: must be owner of function uid (SQLSTATE 42501) -``` - -This happens because the `auth.uid()`, `auth.role()`, and `auth.email()` functions were created by `postgres` (via the init scripts or CloudSync extension), but the auth service runs migrations as `supabase_auth_admin` and expects to own those functions. - -**Symptoms:** -- Auth service keeps restarting -- Supabase Studio shows "Failed to retrieve users" or "column users.banned_until does not exist" (because auth migrations didn't complete) -- `docker compose logs auth` shows the `SQLSTATE 42501` ownership error - -**Fix:** Transfer ownership of the functions to `supabase_auth_admin`: - -```bash -docker compose exec db psql -U postgres -c "ALTER FUNCTION auth.uid() OWNER TO supabase_auth_admin;" -docker compose exec db psql -U postgres -c "ALTER FUNCTION auth.role() OWNER TO supabase_auth_admin;" -docker compose exec db psql -U postgres -c "ALTER FUNCTION auth.email() OWNER TO supabase_auth_admin;" -docker compose restart auth -``` - -After restarting, wait ~30 seconds and check that auth is healthy: - -```bash -docker compose ps auth -docker compose logs --tail=20 auth -``` - -You should see auth in a `healthy` state and the migrations completing successfully. - -### Fix: Supavisor (connection pooler) keeps crashing - -If `docker compose logs supavisor` shows: - -``` -Setting RLIMIT_NOFILE to 100000 -/app/limits.sh: line 6: ulimit: open files: cannot modify limit: Operation not permitted -``` - -This happens because Supavisor's startup script (`/app/limits.sh`) tries to increase the open-file limit to 100,000, but the Fly VM's kernel has a lower cap (typically 10,240) and doesn't allow it. The script failure crashes the container. - -**Fix:** Override the entrypoint in `docker-compose.yml` to skip the limits script. Add this line right after `container_name: supabase-pooler`: - -```yaml - entrypoint: ["/usr/bin/tini", "-s", "-g", "--"] -``` - -Or apply with sed: - -```bash -sed -i '/container_name: supabase-pooler/a\ entrypoint: ["/usr/bin/tini", "-s", "-g", "--"]' /data/supabase-docker/docker-compose.yml -``` - -This keeps the same `tini` init process but skips `/app/limits.sh`. The VM's default open-file limit (10,240) is sufficient for testing. - -Then restart: - -```bash -docker compose up -d supavisor -``` - ---- - -## Step 9: Verify CloudSync - -Connect to the database: - -```bash -docker compose exec db psql -U postgres -``` - -```sql --- Check the extension is installed -SELECT * FROM pg_extension WHERE extname = 'cloudsync'; - --- Check version -SELECT cloudsync_version(); -``` - -If `cloudsync_version()` returns "function does not exist", create the extension manually: - -```sql -CREATE EXTENSION IF NOT EXISTS cloudsync; -``` - ---- - -## Step 10: Test CloudSync Sync - -This section walks through testing the full sync flow: registering the database with the CloudSync server, creating tables, enabling sync, and running a roundtrip test. - -### Prerequisites - -You need a running **CloudSync server**. This can be the staging server or a local instance. - -```bash -export CLOUDSYNC_URL="https://cloudsync-staging-testing.fly.dev" # CloudSync server URL -export ORG_API_KEY="" # Organization API key -``` - -#### Connection string - -The CloudSync server needs a PostgreSQL connection string to reach your database. There are two options depending on where your CloudSync server runs: - -**Option A: CloudSync on the same Fly org (`.internal` network)** - -If both the CloudSync server and the Supabase VM are in the same Fly org, they can communicate over Fly's **private internal network** — no public port exposure needed. Connect directly to the `db` container's mapped port (5432 is exposed on the host by default in docker-compose): - -```bash -# Direct connection (no Supavisor) — recommended for CloudSync server-to-server -export CONNECTION_STRING="postgres://postgres:$POSTGRES_PASSWORD@.internal:5432/postgres" -``` - -**Option B: CloudSync running outside Fly (e.g., local machine, another cloud)** - -Use `fly proxy` to tunnel the Postgres port to your local machine: - -```bash -# In a separate terminal — keep this running -fly proxy 5432:5432 -a -``` - -This makes the remote Postgres available at `localhost:5432`. Then use: - -```bash -export CONNECTION_STRING="postgres://postgres:$POSTGRES_PASSWORD@localhost:5432/postgres" -``` - -> **Note:** The proxy must stay running in a separate terminal for the duration of your session. If the proxy disconnects, just re-run the command. - -To verify the connection works: - -```bash -# Option A: SSH into the VM and test locally -fly ssh console --app -docker compose exec db psql -U postgres -c "SELECT 1;" - -# Option B: With fly proxy running, test from your local machine -psql "postgres://postgres:$POSTGRES_PASSWORD@localhost:5432/postgres" -c "SELECT 1;" -``` - -### 10a. Verify CloudSync server is reachable - -```bash -curl "$CLOUDSYNC_URL/healthz" -# Expected: {"status":"ok"} -``` - -### 10b. Register the Supabase database with CloudSync - -This tells CloudSync where to find your PostgreSQL database: - -```bash -curl --request POST "$CLOUDSYNC_URL/v1/databases" \ - --header "Authorization: Bearer $ORG_API_KEY" \ - --header "Content-Type: application/json" \ - --data '{ - "label": "Supabase Fly.io Test", - "connectionString": "'"$CONNECTION_STRING"'", - "provider": "postgres", - "flavor": "supabase", - "projectId": "cloudsync-supabase-test", - "databaseName": "postgres" - }' -``` - -Save the returned `managedDatabaseId` — you'll need it for all subsequent operations: - -```bash -export MANAGED_DATABASE_ID="" -``` - -### 10c. Verify database connectivity - -```bash -curl --request POST "$CLOUDSYNC_URL/v1/databases/$MANAGED_DATABASE_ID/verify" \ - --header "Authorization: Bearer $ORG_API_KEY" -``` - -Expected: status should show the database is reachable. - -### 10d. Create a test table on the Supabase database - -SSH into the Fly VM and create a table: - -```bash -docker compose exec db psql -U postgres -c " -CREATE TABLE IF NOT EXISTS todos ( - id TEXT PRIMARY KEY DEFAULT cloudsync_uuid(), - title TEXT NOT NULL DEFAULT '', - done BOOLEAN DEFAULT false -); -SELECT cloudsync_init('todos'); -" -``` - -### 10e. Enable CloudSync on the table - -From your local machine, enable sync via the management API: - -```bash -curl --request POST "$CLOUDSYNC_URL/v1/databases/$MANAGED_DATABASE_ID/cloudsync/enable" \ - --header "Authorization: Bearer $ORG_API_KEY" \ - --header "Content-Type: application/json" \ - --data '{"tables":["todos"]}' -``` - -Verify: - -```bash -curl --request GET "$CLOUDSYNC_URL/v1/databases/$MANAGED_DATABASE_ID/cloudsync/tables" \ - --header "Authorization: Bearer $ORG_API_KEY" -``` - -The `todos` table should show `"enabled": true`. - -### 10f. Test sync roundtrip from a SQLite client - -On your local machine, create a SQLite database and sync: - -```sql --- Load the sqlite-sync extension -.load path/to/cloudsync - --- Create the same table schema -CREATE TABLE todos ( - id TEXT PRIMARY KEY DEFAULT (cloudsync_uuid()), - title TEXT NOT NULL DEFAULT '', - done BOOLEAN DEFAULT false -); -SELECT cloudsync_init('todos'); - --- Configure network -SELECT cloudsync_network_init(''); -SELECT cloudsync_network_set_token(''); - --- Insert a row locally -INSERT INTO todos (title) VALUES ('Test from SQLite'); - --- Sync: send local changes, check for remote changes -SELECT cloudsync_network_sync(500, 5); -``` - -Then verify the row arrived on Supabase: - -```bash -docker compose exec db psql -U postgres -c "SELECT * FROM todos;" -``` - -### 10g. Test reverse sync (Supabase → SQLite) - -Insert a row directly on Supabase: - -```bash -docker compose exec db psql -U postgres -c " -INSERT INTO todos (id, title, done) VALUES (cloudsync_uuid(), 'Test from Supabase', false); -" -``` - -Then sync from the SQLite client: - -```sql -SELECT cloudsync_network_check_changes(); -SELECT * FROM todos; -``` - -The row from Supabase should appear in SQLite. - ---- - -## Step 11: Access your services - -| Service | URL | -|---------|-----| -| **Supabase Studio** | `http://:8000` | -| REST API | `http://:8000/rest/v1/` | -| Auth API | `http://:8000/auth/v1/` | -| Storage API | `http://:8000/storage/v1/` | -| Realtime | `http://:8000/realtime/v1/` | - -Studio dashboard requires a username and password. To find them, check your `.env` file on the VM: - -```bash -grep DASHBOARD /data/supabase-docker/.env -``` - -The values are `DASHBOARD_USERNAME` (default: `supabase`) and `DASHBOARD_PASSWORD` (default: `this_password_is_insecure_and_should_be_updated`). - -> **Note:** The Fly VM doesn't expose ports publicly by default. Use `fly proxy` to access services from your local machine: -> ```bash -> fly proxy 8000:8000 -a -> ``` -> Then open `http://localhost:8000` in your browser. - -### Connect to Postgres directly - -Use `fly proxy` to tunnel the Postgres port to your local machine: - -```bash -# In a separate terminal — keep this running -fly proxy 5432:5432 -a -``` - -Then connect from your local machine: - -```bash -psql 'postgres://postgres:@localhost:5432/postgres' -``` - -> **Tip:** You can proxy multiple ports at once by running multiple `fly proxy` commands in separate terminals (e.g., `8000` for Studio and `5432` for Postgres). - ---- - -## Step 11: Set up HTTPS (production) - -For production use, put a reverse proxy in front of Kong. The simplest option is [Caddy](https://caddyserver.com/) which handles TLS automatically. - -On the Fly VM: - -```bash -apt-get install -y debian-keyring debian-archive-keyring apt-transport-https -curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg -curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | tee /etc/apt/sources.list.d/caddy-stable.list -apt-get update -apt-get install -y caddy -``` - -Create `/etc/caddy/Caddyfile`: - -``` -your-domain.com { - reverse_proxy localhost:8000 -} -``` - -```bash -systemctl enable caddy -systemctl start caddy -``` - -Then update `.env`: - -```env -SUPABASE_PUBLIC_URL=https://your-domain.com -API_EXTERNAL_URL=https://your-domain.com -``` - -Restart Supabase: - -```bash -cd /data/supabase-docker -docker compose down && docker compose up -d -``` - ---- - -## Maintenance - -### Update Supabase services - -```bash -cd /data/supabase-docker -# Update image tags in docker-compose.yml, then: -docker compose pull -docker compose down && docker compose up -d -``` - -### Update CloudSync extension - -On your local machine, rebuild and push the image: - -```bash -cd /path/to/sqlite-sync-dev -git pull # get latest CloudSync code -git submodule update --init --recursive # ensure submodules are up to date -make postgres-supabase-build SUPABASE_POSTGRES_TAG=15.8.1.085 -docker tag public.ecr.aws/supabase/postgres:15.8.1.085 \ - /supabase-postgres-cloudsync:15.8.1.085 -docker push /supabase-postgres-cloudsync:15.8.1.085 -``` - -On the Fly VM: - -```bash -cd /data/supabase-docker -docker compose pull db -docker compose up -d db -``` - -### View logs - -```bash -# All services -docker compose logs -f - -# Specific service -docker compose logs -f db -``` - -### Change database password - -```bash -cd /data/supabase-docker -sh ./utils/db-passwd.sh -docker compose up -d --force-recreate -``` - -### Stop and restart the Fly machine - -The Fly machine's root filesystem resets on every stop/start — only the `/data` volume persists. This means Docker must be reinstalled each time. To automate this, create a startup script (run once): - -```bash -cat > /data/startup.sh << 'SCRIPT' -#!/bin/bash -set -e - -echo "=== Installing Docker ===" -apt-get update && apt-get install -y ca-certificates curl gnupg fuse-overlayfs - -install -m 0755 -d /etc/apt/keyrings -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg -chmod a+r /etc/apt/keyrings/docker.gpg - -echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] \ - https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo $VERSION_CODENAME) stable" \ - > /etc/apt/sources.list.d/docker.list - -apt-get update && apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin - -echo "=== Configuring Docker ===" -mkdir -p /etc/docker -echo '{"storage-driver":"fuse-overlayfs","data-root":"/data/docker"}' > /etc/docker/daemon.json - -echo "=== Starting Docker ===" -dockerd > /data/dockerd.log 2>&1 & - -echo "Waiting for Docker to be ready..." -until docker info > /dev/null 2>&1; do sleep 1; done -echo "Docker is ready!" - -echo "=== Starting Supabase ===" -cd /data/supabase-docker -docker compose up -d - -echo "=== Done! ===" -echo "Run 'docker compose ps' to check service status" -SCRIPT -chmod +x /data/startup.sh -``` - -From then on, every time you restart the machine: - -```bash -# From your local machine: -fly machine stop 287920ea023108 -a # stop -fly machine start 287920ea023108 -a # start -fly ssh console --app - -# On the VM — one command does everything: -/data/startup.sh -``` - -Docker images and Supabase data are on `/data`, so they survive restarts. Only Docker itself needs reinstalling (~1-2 minutes). - -### Stop Supabase (without stopping the machine) - -```bash -docker compose down -``` - -### Destroy everything (irreversible) - -```bash -docker compose down -v -rm -rf volumes/db/data volumes/storage -``` - ---- - -## Troubleshooting - -| Problem | Solution | -|---------|----------| -| `fractional_indexing.h: No such file or directory` | Run `git submodule update --init --recursive` before building. The fractional-indexing library is a git submodule that must be initialized. | -| `cloudsync.so: cannot open shared object file` | The custom image wasn't used. Verify `docker compose ps` shows your image, not the default one. | -| Init script didn't run / `cloudsync_version()` not found | Init scripts only run on first start. Run `CREATE EXTENSION IF NOT EXISTS cloudsync;` manually via `docker compose exec db psql -U postgres`. | -| `password authentication failed for user "authenticator"` | Database users have a different password than `.env`. See "Fix: services fail with password authentication failed" above. | -| Analytics (Logflare) keeps crashing | Disable it — see "Fix: analytics keeps crashing" above. It's optional and not needed for CloudSync. | -| `cannot stop container ... did not receive an exit event` | Zombie container. Kill Docker: `kill -9 $(pidof dockerd) $(pidof containerd)`, remove container files: `rm -rf /data/docker/containers/*`, restart dockerd. | -| Services unhealthy after start | Wait 2 minutes. Check `docker compose logs `. Most common: password mismatch (see fix above). | -| Can't pull custom image from VM | Make sure the image is public on Docker Hub, or run `docker login` on the VM. | -| ARM build errors | Fly defaults to x86. If using ARM machines, rebuild with `--platform linux/arm64`. | -| Version mismatch | Run `SHOW server_version;` in psql and ensure your `SUPABASE_POSTGRES_TAG` matches the major version. | -| `no space left on device` during pull | The `fuse-overlayfs` driver is efficient but if disk fills up, extend the volume: `fly volumes extend --size 80 -a `. | -| Docker commands not found after machine restart | The Fly VM root filesystem resets on stop/start. Run `/data/startup.sh` to reinstall Docker. See "Stop and restart the Fly machine" section. | -| Auth crashes with `must be owner of function uid` | Transfer function ownership: `ALTER FUNCTION auth.uid() OWNER TO supabase_auth_admin;` and same for `auth.role()` and `auth.email()`. See "Fix: Auth service crashes" above. | -| Studio shows "column users.banned_until does not exist" | Auth migrations didn't complete. Fix the auth ownership issue above and restart auth. | -| Supavisor crashes with `ulimit: cannot modify limit` | Override entrypoint to skip `/app/limits.sh`. See "Fix: Supavisor keeps crashing" above. | diff --git a/docs/postgresql/quickstarts/supabase-self-hosted.md b/docs/postgresql/quickstarts/supabase-self-hosted.md index 8665191..0e1c910 100644 --- a/docs/postgresql/quickstarts/supabase-self-hosted.md +++ b/docs/postgresql/quickstarts/supabase-self-hosted.md @@ -14,14 +14,20 @@ Follow [Supabase's Installing Supabase](https://supabase.com/docs/guides/self-ho db: # Supabase on PostgreSQL 15 image: sqlitecloud/sqlite-sync-supabase:15 - # instead of: public.ecr.aws/supabase/postgres:15.8.1.085 + # instead of: public.ecr.aws/supabase/postgres:15.8.1.135 # OR Supabase on PostgreSQL 17 image: sqlitecloud/sqlite-sync-supabase:17 # instead of: public.ecr.aws/supabase/postgres:17.6.1.071 + + # OR Supabase on PostgreSQL 17 with a newer Alpine-based Supabase image + image: sqlitecloud/sqlite-sync-supabase:17-alpine + # instead of: public.ecr.aws/supabase/postgres:17.6.1.151 ``` -Use the CloudSync image tag that matches your Supabase PostgreSQL major version. The published major tags `sqlitecloud/sqlite-sync-supabase:15` and `sqlitecloud/sqlite-sync-supabase:17` are the standard choice. Exact Supabase base-image tags may also be published for some releases, but they are optional and not required for normal setup. +Use the CloudSync image tag that matches your Supabase PostgreSQL major version. The published major tags `sqlitecloud/sqlite-sync-supabase:15` and `sqlitecloud/sqlite-sync-supabase:17` are the standard choice. + +**Ubuntu vs Alpine base:** newer Supabase Postgres images (roughly `17.6.1.084` and later) ship an Alpine-based userland instead of the earlier Ubuntu one. The extension itself is identical on both — the PostgreSQL binary is glibc-linked in either case — so for a new deployment the two are interchangeable. Moving an **existing** deployment from one family to the other needs the ownership fix described under [For Existing Deployments](#for-existing-deployments). If your Supabase stack pins one of these newer Alpine base images, use the matching `sqlitecloud/sqlite-sync-supabase:17-alpine` tag (or the exact base tag, e.g. `sqlitecloud/sqlite-sync-supabase:17.6.1.151`) so the CloudSync image is built from the same base. Exact Supabase base-image tags are published for both families but are optional for normal setup. ### Add the CloudSync Init Script @@ -61,6 +67,7 @@ Follow [Supabase's Updating](https://supabase.com/docs/guides/self-hosting/docke # Update docker-compose.yml to use: # sqlitecloud/sqlite-sync-supabase:15 # or sqlitecloud/sqlite-sync-supabase:17 +# or sqlitecloud/sqlite-sync-supabase:17-alpine (newer Alpine-based Supabase bases) docker compose pull docker compose down && docker compose up -d ``` @@ -71,6 +78,31 @@ If Postgres has already been initialized and you are adding CloudSync afterward, CREATE EXTENSION IF NOT EXISTS cloudsync; ``` +#### If Postgres fails to start after moving to a different base image + +Supabase base images do not pin the `postgres` user's UID, so it differs between them — for example `105` on the Ubuntu 20.04 based `15.8.1.085`, `101` on the Ubuntu 24.04 based `15.8.1.135`, and `100` on the Alpine based `17.6.1.151`. A data directory created under one base is unreadable by another, and the container fails to start with: + +``` +cat: /etc/postgresql-custom/pgsodium_root.key: Permission denied +FATAL: invalid secret key +``` + +Re-own the two postgres-owned paths with the UID and GID of the new image: + +```bash +docker compose stop db + +# UID:GID of the postgres user in the new image +docker compose run --rm --no-deps --entrypoint id db postgres + +chown -R : ./volumes/db/data +chown -R : "$(docker volume inspect supabase_db-config --format '{{.Mountpoint}}')" + +docker compose up -d db +``` + +Take a `pg_dumpall` backup first: the change is one-way, and the previous image can no longer read the data directory afterwards. Only deployments whose volumes were initialized under a different base are affected; new deployments are not. + --- ## Step 2: Verify the Extension