From a18635e566f477a677cb7190e85805d6631fe397 Mon Sep 17 00:00:00 2001 From: Xiyuan Chen <52963600+GareArc@users.noreply.github.com> Date: Mon, 15 Jun 2026 02:30:35 -0700 Subject: [PATCH] feat(cli): difyctl per-commit edge distribution via Cloudflare R2 (#37454) --- .github/workflows/cli-edge.yml | 74 +++++++++ cli/README.md | 34 ++-- cli/scripts/install-local.sh | 29 ++-- cli/scripts/install-r2.ps1 | 104 ++++++++++++ cli/scripts/install-r2.ps1.test.ts | 46 ++++++ cli/scripts/install-r2.sh | 164 +++++++++++++++++++ cli/scripts/install-r2.test.ts | 131 +++++++++++++++ cli/scripts/release-naming.mjs | 77 ++++++--- cli/scripts/release-naming.test.ts | 32 ++++ cli/scripts/release-r2-edge.mjs | 129 +++++++++++++++ cli/scripts/release-r2-edge.test.ts | 210 +++++++++++++++++++++++++ cli/scripts/release-r2-publish.sh | 97 ++++++++++++ cli/scripts/release-r2-publish.test.ts | 87 ++++++++++ cli/src/version/info.ts | 2 +- 14 files changed, 1168 insertions(+), 48 deletions(-) create mode 100644 .github/workflows/cli-edge.yml create mode 100644 cli/scripts/install-r2.ps1 create mode 100644 cli/scripts/install-r2.ps1.test.ts create mode 100755 cli/scripts/install-r2.sh create mode 100644 cli/scripts/install-r2.test.ts create mode 100644 cli/scripts/release-r2-edge.mjs create mode 100644 cli/scripts/release-r2-edge.test.ts create mode 100755 cli/scripts/release-r2-publish.sh create mode 100644 cli/scripts/release-r2-publish.test.ts diff --git a/.github/workflows/cli-edge.yml b/.github/workflows/cli-edge.yml new file mode 100644 index 0000000000..159f3545e1 --- /dev/null +++ b/.github/workflows/cli-edge.yml @@ -0,0 +1,74 @@ +name: CLI Edge Publish + +on: + push: + branches: [main] + paths: + - 'cli/**' + - 'packages/contracts/generated/api/openapi/**' + workflow_dispatch: + +concurrency: + group: difyctl-edge-publish + cancel-in-progress: false + +jobs: + publish: + name: build + publish edge to R2 + runs-on: ${{ github.repository == 'langgenius/dify' && 'depot-ubuntu-24.04' || 'ubuntu-24.04' }} + if: vars.DIFYCTL_R2_BUCKET != '' + defaults: + run: + shell: bash + working-directory: ./cli + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + fetch-depth: 0 + + - name: Enable cross-arch native prebuilds + working-directory: ./ + run: cat cli/scripts/cross-arch.pnpm.yaml >> pnpm-workspace.yaml + + - name: Setup web environment + uses: ./.github/actions/setup-web + + - name: Setup Bun + uses: oven-sh/setup-bun@4bc047ad259df6fc24a6c9b0f9a0cb08cf17fbe5 # v2.0.2 + with: + bun-version-file: cli/.bun-version + + - name: Compute edge version + id: ver + run: echo "version=$(node scripts/release-naming.mjs edge-version "$(git rev-parse --short HEAD)")" >> "$GITHUB_OUTPUT" + + - name: Compile standalone binaries (all targets, all-or-nothing) + run: | + CLI_VERSION="${{ steps.ver.outputs.version }}" \ + DIFYCTL_CHANNEL=edge \ + DIFYCTL_COMMIT="$(git rev-parse HEAD)" \ + DIFYCTL_BUILD_DATE="$(git log -1 --format=%cI HEAD)" \ + pnpm build:bin + + - name: Generate sha256 checksums + run: CLI_VERSION="${{ steps.ver.outputs.version }}" scripts/release-write-checksums.sh + + - name: Smoke the runner-arch binary + run: ./dist/bin/difyctl-v${{ steps.ver.outputs.version }}-linux-x64 version + + - name: Publish to R2 + env: + AWS_ACCESS_KEY_ID: ${{ secrets.DIFYCTL_R2_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DIFYCTL_R2_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: auto + AWS_REQUEST_CHECKSUM_CALCULATION: WHEN_REQUIRED + AWS_RESPONSE_CHECKSUM_VALIDATION: WHEN_REQUIRED + DIFYCTL_R2_S3_ENDPOINT: ${{ vars.DIFYCTL_R2_S3_ENDPOINT }} + DIFYCTL_R2_BUCKET: ${{ vars.DIFYCTL_R2_BUCKET }} + DIFYCTL_R2_PUBLIC_BASE: ${{ vars.DIFYCTL_R2_PUBLIC_BASE }} + DIFYCTL_COMMIT: ${{ github.sha }} + run: | + DIFYCTL_BUILD_DATE="$(git log -1 --format=%cI HEAD)" \ + scripts/release-r2-publish.sh edge "${{ steps.ver.outputs.version }}" diff --git a/cli/README.md b/cli/README.md index 61414a2276..8a8521e8d1 100644 --- a/cli/README.md +++ b/cli/README.md @@ -2,25 +2,33 @@ CLI client for [Dify] platform. Browser device-flow signin, list/inspect apps, run with structured input, parse output as JSON, YAML, or human text. -## Install +## Install (edge, internal) -Builds are standalone binaries (Bun-compiled) published as **GitHub Actions workflow artifacts** — no npm, no GitHub Release assets. The installer fetches the latest successful `cli-release.yml` run on `main`, verifies sha256, and copies the binary into `$HOME/.local/bin/difyctl`. +Per-commit `edge` builds are published to Cloudflare R2. The installer script lives in this repo; binaries are fetched from R2 via `DIFYCTL_R2_BASE` (shared internally): ```sh -# GH_TOKEN with `actions:read` scope is required — workflow artifact downloads -# need auth even on public repos. -export GH_TOKEN= -curl -fsSL https://raw.githubusercontent.com/langgenius/dify/main/cli/scripts/install-cli.sh | sh +curl -fsSL https://raw.githubusercontent.com/langgenius/dify/main/cli/scripts/install-r2.sh | DIFYCTL_R2_BASE= sh ``` -| Env | Default | Purpose | -| ---------------- | ----------------- | ----------------------------------------------------- | -| `GH_TOKEN` | — | GitHub PAT (or `GITHUB_TOKEN`) with `actions:read`. | -| `DIFYCTL_PREFIX` | `$HOME/.local` | Install root. Binary lands at `/bin/difyctl`. | -| `DIFYCTL_REPO` | `langgenius/dify` | Source repo. | -| `DIFYCTL_BRANCH` | `main` | Branch to pick the latest successful run from. | +| Env | Default | Purpose | +| ----------------------- | ------------------ | ------------------------------------------------------------------- | +| `DIFYCTL_R2_BASE` | — (required) | R2 public base, e.g. `https://pub-….r2.dev`. | +| `DIFYCTL_CHANNEL` | `edge` | Channel to install. | +| `DIFYCTL_INSTALL_DIR` | `$HOME/.local/bin` | Directory the binary is written to (`/difyctl`). | +| `DIFYCTL_VERSION` | latest | Pin an exact published version. | +| `DIFYCTL_COMMIT` | latest | Pin by git commit (short or full sha). | +| `DIFYCTL_R2_PREFIX` | `difyctl` | R2 key root for the pointer JSONs (`manifest.json` / `index.json`). | +| `DIFYCTL_R2_BIN_PREFIX` | `difyctl/bin` | R2 key root for binaries (the lifecycle/TTL target). | -Supported targets: `darwin-arm64`, `darwin-x64`, `linux-arm64`, `linux-x64`, `windows-x64.exe`. The shell installer covers Linux + macOS; Windows users can download the `.exe` directly from the same artifact. +By default the channel pointer (latest build) is installed. Set `DIFYCTL_COMMIT` (e.g. `ce4af86`) or `DIFYCTL_VERSION` to install a specific past build — both resolve through the channel's `index.json`: + +```sh +curl -fsSL https://raw.githubusercontent.com/langgenius/dify/main/cli/scripts/install-r2.sh | DIFYCTL_R2_BASE= DIFYCTL_COMMIT=ce4af86 sh +``` + +Windows: `$env:DIFYCTL_R2_BASE=''; irm https://raw.githubusercontent.com/langgenius/dify/main/cli/scripts/install-r2.ps1 | iex` (same env vars, e.g. `$env:DIFYCTL_COMMIT='ce4af86'`). + +Re-run to upgrade. For tagged `rc`/`stable` builds, use the GitHub installer (`install-cli.sh`). ## Quickstart diff --git a/cli/scripts/install-local.sh b/cli/scripts/install-local.sh index 3a1b76f78d..892bddf9d9 100755 --- a/cli/scripts/install-local.sh +++ b/cli/scripts/install-local.sh @@ -1,6 +1,10 @@ #!/bin/sh -# install-local.sh — install difyctl from a locally built tarball. -# Run via: pnpm install:local +# install-local.sh — install difyctl from locally built standalone binaries. +# Run via: pnpm install:local (after `pnpm build:bin`) +# +# Consumes the raw, self-contained binaries emitted by scripts/release-build.sh +# into dist/bin (difyctl-v--). No GitHub Release needed: build on +# one machine, copy dist/bin to the tester, point this script at that directory. set -eu PREFIX="${DIFYCTL_PREFIX:-${HOME}/.local}" @@ -10,7 +14,7 @@ BIN_DIR="${PREFIX}/bin" case "$(uname -s)" in Linux*) os=linux ;; Darwin*) os=darwin ;; - *) echo "unsupported OS: $(uname -s)" >&2; exit 1 ;; + *) echo "unsupported OS: $(uname -s) (use install.ps1 on Windows)" >&2; exit 1 ;; esac case "$(uname -m)" in @@ -20,20 +24,21 @@ case "$(uname -m)" in esac # Accept an optional directory path as the first argument. -# Default to the cli/dist directory if not provided. -ARTIFACT_DIR="${1:-$(cd "$(dirname "$0")/../dist" && pwd)}" -TARBALL="$(ls "${ARTIFACT_DIR}"/difyctl-*-${os}-${arch}.tar.xz 2>/dev/null | head -1)" +# Default to the cli/dist/bin directory if not provided. +ARTIFACT_DIR="${1:-$(cd "$(dirname "$0")/../dist/bin" 2>/dev/null && pwd || true)}" +BINARY="$(ls "${ARTIFACT_DIR}"/difyctl-v*-${os}-${arch} 2>/dev/null | sort -V | tail -1)" -if [ -z "$TARBALL" ]; then - echo "no tarball found for ${os}-${arch} in ${ARTIFACT_DIR}" >&2 - echo "run: pnpm pack:tarballs" >&2 +if [ -z "$BINARY" ]; then + echo "no binary found for ${os}-${arch} in ${ARTIFACT_DIR:-}" >&2 + echo "run: pnpm build:bin" >&2 exit 1 fi -echo "installing from $(basename "$TARBALL") ..." +echo "installing from $(basename "$BINARY") ..." rm -rf "$SHARE_DIR" -mkdir -p "$SHARE_DIR" "$BIN_DIR" -tar -xJf "$TARBALL" -C "$SHARE_DIR" --strip-components=1 +mkdir -p "${SHARE_DIR}/bin" "$BIN_DIR" +cp "$BINARY" "${SHARE_DIR}/bin/difyctl" +chmod +x "${SHARE_DIR}/bin/difyctl" ln -sf "${SHARE_DIR}/bin/difyctl" "${BIN_DIR}/difyctl" echo "installed: ${BIN_DIR}/difyctl" diff --git a/cli/scripts/install-r2.ps1 b/cli/scripts/install-r2.ps1 new file mode 100644 index 0000000000..44321d13fa --- /dev/null +++ b/cli/scripts/install-r2.ps1 @@ -0,0 +1,104 @@ +# install-r2.ps1 — one-line difyctl installer (Windows) from Cloudflare R2. +# Usage: $env:DIFYCTL_R2_BASE=''; irm https://raw.githubusercontent.com/langgenius/dify/main/cli/scripts/install-r2.ps1 | iex +# Env: DIFYCTL_R2_BASE (required), DIFYCTL_CHANNEL (default edge), +# DIFYCTL_INSTALL_DIR (default %LOCALAPPDATA%\difyctl\bin; binary written here as difyctl.exe), +# DIFYCTL_VERSION (pin exact published version), +# DIFYCTL_COMMIT (pin by git commit, short or full sha; via index.json), +# DIFYCTL_R2_PREFIX (default difyctl; key root for pointer JSONs), +# DIFYCTL_R2_BIN_PREFIX (default /bin; key root for binaries). +# With no pin the channel pointer (latest) is installed. +$ErrorActionPreference = 'Stop' + +function Get-TargetField($manifest, [string]$target, [string]$field) { + $t = $manifest.targets.$target + if (-not $t) { return $null } + return $t.$field +} + +# First build in index.json matching by exact version or commit prefix. +function Resolve-IndexBuild($index, [string]$kind, [string]$want) { + foreach ($b in $index.builds) { + $sel = if ($kind -eq 'commit') { $b.commit } else { $b.version } + $hit = if ($kind -eq 'commit') { $sel.StartsWith($want) } else { $sel -eq $want } + if ($hit) { return $b } + } + return $null +} + +# Parse a checksums.txt (" ") for the line whose asset matches the +# target; returns @{ Sha; Asset } or $null. +function Get-ChecksumTarget([string]$text, [string]$target) { + foreach ($line in ($text -split "`n")) { + $line = $line.Trim() + if ($line -match "\sdifyctl-v.*-$target(\.exe)?$") { + $parts = $line -split '\s+' + return @{ Sha = $parts[0]; Asset = $parts[-1] } + } + } + return $null +} + +# Download, sha256-verify, place. Returns nothing; throws on mismatch. +function Install-DifyctlBinary([string]$dlUrl, [string]$sha, [string]$version, [string]$channel, [string]$installDir) { + $tmp = Join-Path ([System.IO.Path]::GetTempPath()) ([System.IO.Path]::GetFileName($dlUrl)) + Invoke-WebRequest -Uri $dlUrl -OutFile $tmp + $actual = (Get-FileHash -Path $tmp -Algorithm SHA256).Hash.ToLower() + if ($actual -ne $sha.ToLower()) { throw "checksum mismatch for $dlUrl" } + + New-Item -ItemType Directory -Path $installDir -Force | Out-Null + $dest = Join-Path $installDir 'difyctl.exe' + try { Move-Item -Path $tmp -Destination $dest -Force } + catch { throw "cannot replace $dest — close any running difyctl and re-run." } + + Write-Host "difyctl $version (channel $channel) installed: $dest" + if (($env:PATH -split ';') -notcontains $installDir) { + Write-Host "$installDir is not on your PATH. Add it with:" + Write-Host " [Environment]::SetEnvironmentVariable('PATH', `"$installDir;`$env:PATH`", 'User')" + } +} + +function Install-DifyctlR2 { + $base = $env:DIFYCTL_R2_BASE + if (-not $base) { throw "set DIFYCTL_R2_BASE to the R2 public base (e.g. https://pub-….r2.dev)" } + $base = $base.TrimEnd('/') + $channel = if ($env:DIFYCTL_CHANNEL) { $env:DIFYCTL_CHANNEL } else { 'edge' } + $prefix = if ($env:DIFYCTL_R2_PREFIX) { $env:DIFYCTL_R2_PREFIX } else { 'difyctl' } + $binPrefix = if ($env:DIFYCTL_R2_BIN_PREFIX) { $env:DIFYCTL_R2_BIN_PREFIX } else { "$prefix/bin" } + $installDir = if ($env:DIFYCTL_INSTALL_DIR) { $env:DIFYCTL_INSTALL_DIR } else { Join-Path (Join-Path $env:LOCALAPPDATA 'difyctl') 'bin' } + $target = 'windows-x64' + + if ($env:DIFYCTL_VERSION -or $env:DIFYCTL_COMMIT) { + $iurl = "$base/$prefix/$channel/index.json" + try { $index = Invoke-RestMethod -Uri $iurl } + catch { throw "R2 unavailable fetching $iurl; retry." } + $build = if ($env:DIFYCTL_VERSION) { Resolve-IndexBuild $index 'version' $env:DIFYCTL_VERSION } + else { Resolve-IndexBuild $index 'commit' $env:DIFYCTL_COMMIT } + $pin = if ($env:DIFYCTL_VERSION) { $env:DIFYCTL_VERSION } else { $env:DIFYCTL_COMMIT } + if (-not $build) { throw "no build matching $pin in channel $channel" } + $version = $build.version + $vbase = "$base/$binPrefix/$channel/$($build.dir)" + try { $cf = (Invoke-WebRequest -Uri "$vbase/difyctl-v$version-checksums.txt").Content } + catch { throw "checksums missing for $version (channel $channel)" } + $ct = Get-ChecksumTarget $cf $target + if (-not $ct) { throw "no build for $target at $version" } + Install-DifyctlBinary "$vbase/$($ct.Asset)" $ct.Sha $version $channel $installDir + return + } + + $murl = "$base/$prefix/$channel/manifest.json" + try { $manifest = Invoke-RestMethod -Uri $murl } + catch { + if ($_.Exception.Response.StatusCode.value__ -eq 404) { + throw "channel '$channel' not published to R2. For rc/stable use install.ps1 (GitHub)." + } + throw "R2 unavailable fetching $murl; retry." + } + if ($manifest.channel -ne $channel) { throw "manifest channel '$($manifest.channel)' != requested '$channel'" } + + $asset = Get-TargetField $manifest $target 'asset' + $sha = Get-TargetField $manifest $target 'sha256' + if (-not $asset) { throw "no build for $target in channel $channel" } + Install-DifyctlBinary "$($manifest.baseUrl)/$asset" $sha $manifest.version $channel $installDir +} + +if ($env:DIFYCTL_INSTALL_LIB -ne '1') { Install-DifyctlR2 } diff --git a/cli/scripts/install-r2.ps1.test.ts b/cli/scripts/install-r2.ps1.test.ts new file mode 100644 index 0000000000..672f103c4a --- /dev/null +++ b/cli/scripts/install-r2.ps1.test.ts @@ -0,0 +1,46 @@ +import { spawnSync } from 'node:child_process' +import { fileURLToPath } from 'node:url' +import { describe, expect, it } from 'vitest' + +const SCRIPT = fileURLToPath(new URL('./install-r2.ps1', import.meta.url)) +const hasPwsh = spawnSync('pwsh', ['-v'], { encoding: 'utf8' }).status === 0 +const d = hasPwsh ? describe : describe.skip + +const MANIFEST = JSON.stringify({ + schema: 1, + name: 'difyctl', + channel: 'edge', + version: '0.1.0-edge.2fd7b82', + baseUrl: 'https://pub.example.r2.dev/difyctl/edge/0.1.0-edge.2fd7b82', + targets: { 'windows-x64': { asset: 'difyctl-v0.1.0-edge.2fd7b82-windows-x64.exe', sha256: 'deadbeef' } }, +}) + +function pwsh(program: string): { code: number, stdout: string, stderr: string } { + const full = `. '${SCRIPT}'\n${program}` + const r = spawnSync('pwsh', ['-NoProfile', '-Command', full], { + encoding: 'utf8', + env: { ...process.env, DIFYCTL_INSTALL_LIB: '1' }, + }) + return { code: r.status ?? 1, stdout: (r.stdout ?? '').replace(/\r\n/g, '\n').trim(), stderr: r.stderr ?? '' } +} + +d('install-r2.ps1', () => { + it('parses a target asset + sha from the manifest', () => { + const prog = `$m = ConvertFrom-Json @'\n${MANIFEST}\n'@\n` + + `Write-Output (Get-TargetField $m 'windows-x64' 'asset')\n` + + `Write-Output (Get-TargetField $m 'windows-x64' 'sha256')` + const { stdout } = pwsh(prog) + expect(stdout).toBe('difyctl-v0.1.0-edge.2fd7b82-windows-x64.exe\ndeadbeef') + }) + + it('errors when DIFYCTL_R2_BASE is unset', () => { + const r = spawnSync('pwsh', ['-NoProfile', '-File', SCRIPT], { + encoding: 'utf8', + env: { ...process.env, DIFYCTL_R2_BASE: '' }, + }) + if (hasPwsh) { + expect(r.status).not.toBe(0) + expect(r.stderr + r.stdout).toMatch(/DIFYCTL_R2_BASE/) + } + }) +}) diff --git a/cli/scripts/install-r2.sh b/cli/scripts/install-r2.sh new file mode 100755 index 0000000000..ffb4d98042 --- /dev/null +++ b/cli/scripts/install-r2.sh @@ -0,0 +1,164 @@ +#!/bin/sh +# install-r2.sh — one-line difyctl installer from Cloudflare R2. +# Reads a per-channel pointer manifest, sha256-verifies, installs to PATH. +# Usage: curl -fsSL https://raw.githubusercontent.com/langgenius/dify/main/cli/scripts/install-r2.sh | DIFYCTL_R2_BASE= sh +# Env: +# DIFYCTL_R2_BASE (required) R2 public base, e.g. https://pub-….r2.dev +# DIFYCTL_CHANNEL (default edge) +# DIFYCTL_INSTALL_DIR (default $HOME/.local/bin) directory the binary is written to as /difyctl +# DIFYCTL_VERSION pin an exact published version (e.g. 0.1.0-edge.ce4af868) +# DIFYCTL_COMMIT pin by git commit (short or full sha); resolved via index.json +# DIFYCTL_R2_PREFIX (default difyctl) key root for pointer JSONs +# DIFYCTL_R2_BIN_PREFIX (default /bin) key root for binaries +# With no pin the channel pointer (latest) is installed. A pin resolves through +# //index.json -> the build's immutable dir under the bin prefix. +set -eu + +# --- library functions (sourced for tests when DIFYCTL_INSTALL_LIB=1) --- +tmp_m="$(mktemp 2>/dev/null || echo /tmp/difyctl-manifest.$$)" +trap 'rm -f "$tmp_m" "${tmp_c:-}" "${tmp_b:-}"' EXIT INT TERM + +err() { printf '%s\n' "install-r2: $*" >&2; } +die() { err "$*"; exit 1; } +need() { command -v "$1" >/dev/null 2>&1 || die "$1 is required"; } + +detect_target() { + case "$(uname -s)" in + Linux*) _os=linux ;; + Darwin*) _os=darwin ;; + *) die "unsupported OS: $(uname -s) (use install.ps1 on Windows)" ;; + esac + case "$(uname -m)" in + x86_64|amd64) _arch=x64 ;; + arm64|aarch64) _arch=arm64 ;; + *) die "unsupported arch: $(uname -m)" ;; + esac + printf '%s-%s' "$_os" "$_arch" +} + +# grep/sed (no jq). Correct only because release-r2-edge.mjs renders one key per line. +# manifest_str -> value of a top-level string field +manifest_str() { + grep "\"$2\"[[:space:]]*:" "$1" | head -1 \ + | sed -E "s/.*\"$2\"[[:space:]]*:[[:space:]]*\"([^\"]+)\".*/\\1/" +} + +# manifest_target_field -> value on that target's line +manifest_target_field() { + grep "\"$2\"[[:space:]]*:" "$1" | head -1 \ + | sed -E "s/.*\"$3\"[[:space:]]*:[[:space:]]*\"([^\"]+)\".*/\\1/" +} + +# Resolve a pinned build from index.json (no jq). Correct only because +# release-r2-edge.mjs renders each build's fields one per line, dir last. +# Prints "\t" of the first match, nothing if none. +# index_resolve (commit = prefix match) +index_resolve() { + awk -v kind="$2" -v want="$3" ' + function val(s) { sub(/^[^:]*:[[:space:]]*"/, "", s); sub(/".*$/, "", s); return s } + /"version"[[:space:]]*:/ { v = val($0) } + /"commit"[[:space:]]*:/ { c = val($0) } + /"dir"[[:space:]]*:/ { + d = val($0) + sel = (kind == "commit") ? c : v + if (kind == "commit") { if (index(sel, want) == 1) { print v "\t" d; exit } } + else if (sel == want) { print v "\t" d; exit } + } + ' "$1" +} + +# checksums_target -> "\t" +# checksums lines are " "; match asset ending - or -.exe. +checksums_target() { + grep -E "[[:space:]]difyctl-v.*-$2(\.exe)?\$" "$1" | head -1 \ + | awk '{ print $1 "\t" $NF }' +} + +sha256_check() { + # $1 = file, $2 = expected hex + if command -v sha256sum >/dev/null 2>&1; then _a="$(sha256sum "$1" | awk '{print $1}')" + elif command -v shasum >/dev/null 2>&1; then _a="$(shasum -a 256 "$1" | awk '{print $1}')" + else die "no sha256 tool (need sha256sum or shasum)"; fi + [ "$_a" = "$2" ] || die "checksum mismatch for $1" +} + +# fetch_verify_install +fetch_verify_install() { + tmp_b="$(mktemp)" + # NOTE: no --compressed — must hash the raw bytes + curl -fsSL "$1" -o "$tmp_b" || die "download failed: $1" + sha256_check "$tmp_b" "$2" + + mkdir -p "$install_dir" + chmod +x "$tmp_b" + mv "$tmp_b" "${install_dir}/difyctl" 2>/dev/null || { cp "$tmp_b" "${install_dir}/difyctl"; rm -f "$tmp_b"; } + + printf 'difyctl %s (channel %s) installed: %s\n' "$3" "$4" "${install_dir}/difyctl" + case ":${PATH}:" in + *":${install_dir}:"*) ;; + *) printf 'note: add %s to your PATH\n' "$install_dir" ;; + esac +} + +# Resolve a pinned build into download url + sha. Sets: version, dl_url, dl_sha. +resolve_pinned() { + iurl="${base}/${prefix}/${channel}/index.json" + curl -fsSL "$iurl" -o "$tmp_m" || die "R2 unavailable fetching ${iurl}; retry." + if [ -n "${DIFYCTL_VERSION:-}" ]; then res="$(index_resolve "$tmp_m" version "$DIFYCTL_VERSION")" + else res="$(index_resolve "$tmp_m" commit "$DIFYCTL_COMMIT")"; fi + [ -n "$res" ] || die "no build matching ${DIFYCTL_VERSION:-$DIFYCTL_COMMIT} in channel ${channel}" + version="$(printf '%s' "$res" | cut -f1)" + dir="$(printf '%s' "$res" | cut -f2)" + + vbase="${base}/${bin_prefix}/${channel}/${dir}" + tmp_c="$(mktemp)" + curl -fsSL "${vbase}/difyctl-v${version}-checksums.txt" -o "$tmp_c" \ + || die "checksums missing for ${version} (channel ${channel})" + line="$(checksums_target "$tmp_c" "$target")" + [ -n "$line" ] || die "no build for ${target} at ${version}" + dl_sha="$(printf '%s' "$line" | cut -f1)" + dl_url="${vbase}/$(printf '%s' "$line" | cut -f2)" +} + +# Resolve the channel pointer (latest) into download url + sha. Sets the same. +resolve_pointer() { + murl="${base}/${prefix}/${channel}/manifest.json" + _code="$(curl -fsS -o "$tmp_m" -w '%{http_code}' "$murl" 2>/dev/null || true)" + if [ ! -s "$tmp_m" ]; then + case "$_code" in + 404) die "channel '${channel}' not published to R2. For rc/stable use the GitHub installer (install-cli.sh)." ;; + *) die "R2 unavailable (HTTP ${_code:-?}) fetching ${murl}; retry." ;; + esac + fi + mchannel="$(manifest_str "$tmp_m" channel)" + [ "$mchannel" = "$channel" ] || die "manifest channel '${mchannel}' != requested '${channel}'" + version="$(manifest_str "$tmp_m" version)" + baseUrl="$(manifest_str "$tmp_m" baseUrl)" + asset="$(manifest_target_field "$tmp_m" "$target" asset)" + dl_sha="$(manifest_target_field "$tmp_m" "$target" sha256)" + [ -n "$asset" ] && [ -n "$dl_sha" ] || die "no build for ${target} in channel ${channel}" + dl_url="${baseUrl}/${asset}" +} + +install_main() { + need curl + [ -n "${DIFYCTL_R2_BASE:-}" ] || die "set DIFYCTL_R2_BASE to the R2 public base (e.g. https://pub-….r2.dev)" + base="${DIFYCTL_R2_BASE%/}" + channel="${DIFYCTL_CHANNEL:-edge}" + prefix="${DIFYCTL_R2_PREFIX:-difyctl}" + bin_prefix="${DIFYCTL_R2_BIN_PREFIX:-${prefix}/bin}" + install_dir="${DIFYCTL_INSTALL_DIR:-${HOME}/.local/bin}" + target="$(detect_target)" + + if [ -n "${DIFYCTL_VERSION:-}" ] || [ -n "${DIFYCTL_COMMIT:-}" ]; then + resolve_pinned + else + resolve_pointer + fi + + fetch_verify_install "$dl_url" "$dl_sha" "$version" "$channel" +} + +if [ "${DIFYCTL_INSTALL_LIB:-0}" != "1" ]; then + install_main +fi diff --git a/cli/scripts/install-r2.test.ts b/cli/scripts/install-r2.test.ts new file mode 100644 index 0000000000..3e02439d6c --- /dev/null +++ b/cli/scripts/install-r2.test.ts @@ -0,0 +1,131 @@ +import { spawnSync } from 'node:child_process' +import { fileURLToPath } from 'node:url' +import { describe, expect, it } from 'vitest' + +const SCRIPT = fileURLToPath(new URL('./install-r2.sh', import.meta.url)) + +const MANIFEST = [ + '{', + ' "schema": 1,', + ' "name": "difyctl",', + ' "channel": "edge",', + ' "version": "0.1.0-edge.2fd7b82",', + ' "commit": "abc1234",', + ' "buildDate": "2026-06-14T12:00:00Z",', + ' "compat": {"minDify":"1.14.0","maxDify":"1.15.0"},', + ' "baseUrl": "https://pub.example.r2.dev/difyctl/edge/0.1.0-edge.2fd7b82",', + ' "targets": {', + ' "linux-x64": { "asset": "difyctl-v0.1.0-edge.2fd7b82-linux-x64", "sha256": "deadbeef" },', + ' "darwin-arm64": { "asset": "difyctl-v0.1.0-edge.2fd7b82-darwin-arm64", "sha256": "cafef00d" }', + ' }', + '}', +].join('\n') + +const INDEX = [ + '{', + ' "schema": 1,', + ' "channel": "edge",', + ' "updated": "2026-06-15T00:00:00Z",', + ' "builds": [', + ' {', + ' "version": "0.1.0-edge.ce4af868",', + ' "commit": "ce4af868d653f405070fabb3be3303430cc030ad",', + ' "buildDate": "2026-06-15T00:00:00Z",', + ' "dir": "0.1.0-edge.ce4af868"', + ' },', + ' {', + ' "version": "0.1.0-edge.aaaa111",', + ' "commit": "aaaa111bbbbcccc000011112222333344445555",', + ' "buildDate": "2026-06-14T00:00:00Z",', + ' "dir": "0.1.0-edge.aaaa111"', + ' }', + ' ]', + '}', +].join('\n') + +const CHECKSUMS = [ + 'deadbeef difyctl-v0.1.0-edge.ce4af868-linux-x64', + 'cafef00d difyctl-v0.1.0-edge.ce4af868-darwin-arm64', + 'beadc0de difyctl-v0.1.0-edge.ce4af868-windows-x64.exe', +].join('\n') + +function lib(program: string, env: Record = {}): { code: number, stdout: string, stderr: string } { + const full = `. "${SCRIPT}"\n${program}` + const r = spawnSync('sh', ['-c', full], { + encoding: 'utf8', + env: { ...process.env, DIFYCTL_INSTALL_LIB: '1', ...env }, + }) + return { code: r.status ?? 1, stdout: (r.stdout ?? '').trim(), stderr: r.stderr ?? '' } +} + +describe('install-r2 manifest parsing', () => { + // install-r2.sh is POSIX-only; under git-bash on Windows `uname -s` is MINGW*, + // so detect_target intentionally dies (Windows installs go through install-r2.ps1). + it.skipIf(process.platform === 'win32')('detect_target maps to one of the 5 ids', () => { + const { stdout } = lib('detect_target') + expect(['linux-x64', 'linux-arm64', 'darwin-x64', 'darwin-arm64', 'windows-x64']).toContain(stdout) + }) + + it('manifest_str reads a top-level string field', () => { + const { stdout } = lib(`printf '%s' '${MANIFEST}' > "$tmp_m"; manifest_str "$tmp_m" channel`, {}) + expect(stdout).toBe('edge') + }) + + it('manifest_target_field extracts per-target values from a single line', () => { + const prog = `printf '%s' '${MANIFEST}' > "$tmp_m"\n` + + 'manifest_target_field "$tmp_m" darwin-arm64 asset\n' + + 'manifest_target_field "$tmp_m" darwin-arm64 sha256' + const { stdout } = lib(prog) + expect(stdout).toBe('difyctl-v0.1.0-edge.2fd7b82-darwin-arm64\ncafef00d') + }) + + it('requires DIFYCTL_R2_BASE when run as the installer (not lib)', () => { + const r = spawnSync('sh', [SCRIPT], { encoding: 'utf8', env: { ...process.env, DIFYCTL_R2_BASE: '' } }) + expect(r.status).not.toBe(0) + expect(r.stderr).toMatch(/DIFYCTL_R2_BASE/) + }) + + it('sha256_check aborts on a checksum mismatch', () => { + const r = lib('f="$(mktemp)"; printf \'hello\' > "$f"; sha256_check "$f" deadbeef') + expect(r.code).not.toBe(0) + expect(r.stderr).toMatch(/checksum mismatch/) + }) + + it('sha256_check passes on the correct hash', () => { + // sha256('hello') = 2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 + const r = lib('f="$(mktemp)"; printf \'hello\' > "$f"; sha256_check "$f" 2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 && echo OK') + expect(r.stdout).toBe('OK') + }) +}) + +describe('install-r2 version/commit pin', () => { + it('index_resolve matches a build by exact version', () => { + const r = lib(`printf '%s' '${INDEX}' > "$tmp_m"; index_resolve "$tmp_m" version 0.1.0-edge.aaaa111`) + expect(r.stdout).toBe('0.1.0-edge.aaaa111\t0.1.0-edge.aaaa111') + }) + + it('index_resolve matches a build by commit prefix', () => { + const r = lib(`printf '%s' '${INDEX}' > "$tmp_m"; index_resolve "$tmp_m" commit ce4af868`) + expect(r.stdout).toBe('0.1.0-edge.ce4af868\t0.1.0-edge.ce4af868') + }) + + it('index_resolve matches the full 40-char commit too', () => { + const r = lib(`printf '%s' '${INDEX}' > "$tmp_m"; index_resolve "$tmp_m" commit aaaa111bbbbcccc000011112222333344445555`) + expect(r.stdout).toBe('0.1.0-edge.aaaa111\t0.1.0-edge.aaaa111') + }) + + it('index_resolve prints nothing when no build matches', () => { + const r = lib(`printf '%s' '${INDEX}' > "$tmp_m"; index_resolve "$tmp_m" commit ffffff`) + expect(r.stdout).toBe('') + }) + + it('checksums_target extracts sha and asset for a posix target', () => { + const r = lib(`printf '%s' '${CHECKSUMS}' > "$tmp_m"; checksums_target "$tmp_m" darwin-arm64`) + expect(r.stdout).toBe('cafef00d\tdifyctl-v0.1.0-edge.ce4af868-darwin-arm64') + }) + + it('checksums_target does not bleed x64 into arm64', () => { + const r = lib(`printf '%s' '${CHECKSUMS}' > "$tmp_m"; checksums_target "$tmp_m" linux-x64`) + expect(r.stdout).toBe('deadbeef\tdifyctl-v0.1.0-edge.ce4af868-linux-x64') + }) +}) diff --git a/cli/scripts/release-naming.mjs b/cli/scripts/release-naming.mjs index 17ce210222..5d7457b59b 100644 --- a/cli/scripts/release-naming.mjs +++ b/cli/scripts/release-naming.mjs @@ -17,19 +17,17 @@ // validate -> exit 1 if difyctl.release, version, or channel is malformed // compat-check -> exit 1 if difyVer outside compat.minDify..maxDify -import { readFileSync } from 'node:fs' +import { readFileSync, realpathSync } from 'node:fs' +import { fileURLToPath } from 'node:url' const BUN_TARGET_RE = /^bun-(linux|darwin|windows)-(x64|arm64)$/ const SEMVER_CORE_LEN = 3 -// Channel registry — single source for which version forms are releasable and -// resolvable. Each `versionForm` is pinned to exactly what the installers' -// channel filters accept (stable = no prerelease; rc = -rc.N with nothing -// trailing), so any version that passes `validate` is guaranteed resolvable at -// install time. Extend by adding an entry: { name, prerelease, versionForm }. +// Add channels here: { name, prerelease, versionForm }. const CHANNELS = [ { name: 'stable', prerelease: false, versionForm: /^\d+\.\d+\.\d+(\+[0-9A-Z.-]+)?$/i }, { name: 'rc', prerelease: true, versionForm: /^\d+\.\d+\.\d+-rc\.\d+$/ }, + { name: 'edge', prerelease: true, versionForm: /^\d+\.\d+\.\d+-edge\.[0-9a-f]{7,40}$/ }, ] const channelByName = name => CHANNELS.find(c => c.name === name) @@ -43,6 +41,41 @@ function parsePrecedence(v) { return { nums: core.split('.').map(Number), pre } } +function versionCore(v) { + return String(v).replace(/^v/, '').replace(/\+.*$/, '').split('-')[0] +} + +function edgeVersion(sha) { + if (!/^[0-9a-f]{7,40}$/.test(sha ?? '')) + die('edge-version requires a git short sha (7-40 hex chars)') + const { version } = loadPkg() + const core = versionCore(version) + if (!/^\d+\.\d+\.\d+$/.test(core)) + die(`cannot derive edge base from version: ${version}`) + return `${core}-edge.${sha}` +} + +// Returns a problem string if `version` cannot be resolved under `channel`, else +// null. Shared by validateVersionForChannel (die-now) and validateVersionChannel +// (collect for the `validate` gate). +function channelVersionProblem(version, channel) { + if (typeof version !== 'string' || version.length === 0) + return 'version must be a non-empty string' + const ch = channelByName(channel) + if (!ch) + return `unknown channel: ${channel} (expected one of: ${channelNames()})` + if (!ch.versionForm.test(version)) + return `version ${version} does not match the ${channel} channel form` + return null +} + +function validateVersionForChannel(version, channelName) { + const problem = channelVersionProblem(version, channelName) + if (problem) + die(problem) + return `valid: ${version} is a ${channelName} version` +} + function comparePre(a, b) { const aparts = a.split('.') const bparts = b.split('.') @@ -105,9 +138,7 @@ function loadPkg() { } } -// Every field downstream CI needs, as `key=value` lines for $GITHUB_ENV. Each -// job pipes this once into the environment, then references ${{ env. }} -// at use sites. +// Emits key=value lines for $GITHUB_ENV. function githubEnv() { const { version, channel, compat, release } = loadPkg() const fields = { @@ -167,19 +198,9 @@ function validateRelease(release) { return problems } -// Enforce that the version matches the form its declared channel can resolve. -// Rejects e.g. channel=rc + 1.2.3-rc5 (no dot), channel=stable + 1.2.3-rc.1, -// or any unknown channel — before a release that no installer could find ships. function validateVersionChannel(version, channel) { - const problems = [] - if (typeof version !== 'string' || version.length === 0) - return ['package.json version must be a non-empty string'] - const ch = channelByName(channel) - if (!ch) - return [`difyctl.channel ${JSON.stringify(channel)} is not a known channel (expected one of: ${channelNames()})`] - if (!ch.versionForm.test(version)) - problems.push(`version "${version}" does not match the ${channel} channel form ${ch.versionForm}; an installer could not resolve it`) - return problems + const problem = channelVersionProblem(version, channel) + return problem ? [problem] : [] } function main(argv) { @@ -223,9 +244,21 @@ function main(argv) { die(`invalid difyctl release config:\n - ${problems.join('\n - ')}`) return `difyctl release valid: version=${version} channel=${channel} targets=${release.targets.length}` } + case 'edge-version': + return edgeVersion(rest[0]) + case 'validate-version': + return validateVersionForChannel( + requireVersion(rest[0]), + rest[1] ?? die('channel argument is required'), + ) default: die(`unknown subcommand: ${cmd ?? '(none)'}`) } } -process.stdout.write(`${main(process.argv.slice(2))}\n`) +const invokedDirectly = process.argv[1] + && realpathSync(process.argv[1]) === fileURLToPath(import.meta.url) +if (invokedDirectly) + process.stdout.write(`${main(process.argv.slice(2))}\n`) + +export { assetName, channelByName, CHANNELS, edgeVersion, loadPkg, validateVersionForChannel, versionCore } diff --git a/cli/scripts/release-naming.test.ts b/cli/scripts/release-naming.test.ts index bf971db50e..ad2a699ef8 100644 --- a/cli/scripts/release-naming.test.ts +++ b/cli/scripts/release-naming.test.ts @@ -73,3 +73,35 @@ describe('release-naming github-env', () => { expect(stdout).toMatch(new RegExp(`^${key}=`, 'm')) }) }) + +describe('release-naming edge channel', () => { + it('lists edge among channels', () => { + expect(run(['channels']).stdout).toMatch(/^edge$/m) + }) + + it('edge-version derives -edge. stripping the rc prerelease', () => { + // package.json version is 0.1.0-rc.1 -> core 0.1.0 + expect(run(['edge-version', '2fd7b82']).stdout.trim()).toBe('0.1.0-edge.2fd7b82') + }) + + it('edge-version accepts a 40-char sha', () => { + const sha = '2fd7b829e1f0aaaabbbbccccddddeeeeffff0000' + expect(run(['edge-version', sha]).stdout.trim()).toBe(`0.1.0-edge.${sha}`) + }) + + it('edge-version rejects a non-hex sha', () => { + expect(run(['edge-version', 'nothex!']).code).not.toBe(0) + }) + + it('edge-version requires a sha argument', () => { + expect(run(['edge-version']).code).not.toBe(0) + }) + + it('the edge version form matches a computed edge version', () => { + expect(run(['validate-version', '0.1.0-edge.2fd7b82', 'edge']).code).toBe(0) + }) + + it('validate-version rejects an rc string under the edge channel', () => { + expect(run(['validate-version', '0.1.0-rc.1', 'edge']).code).not.toBe(0) + }) +}) diff --git a/cli/scripts/release-r2-edge.mjs b/cli/scripts/release-r2-edge.mjs new file mode 100644 index 0000000000..55b40435b6 --- /dev/null +++ b/cli/scripts/release-r2-edge.mjs @@ -0,0 +1,129 @@ +#!/usr/bin/env node +// release-r2-edge.mjs — edge/R2 release metadata generator. Two subcommands: +// manifest -> the per-channel pointer manifest.json (the installer reads this) +// index -> the per-channel build-history ledger index.json +import { existsSync, readFileSync } from 'node:fs' +import { assetName, loadPkg, validateVersionForChannel } from './release-naming.mjs' + +function die(msg) { + process.stderr.write(`release-r2-edge: ${msg}\n`) + process.exit(1) +} + +function parseArgs(argv) { + const out = {} + for (let i = 0; i < argv.length; i += 2) { + const key = argv[i]?.replace(/^--/, '') + const val = argv[i + 1] + if (!key || val === undefined || val.startsWith('--')) + die(`malformed argument near ${argv[i]} (expected --key value)`) + out[key] = val + } + return out +} + +function requireArgs(args, keys) { + for (const k of keys) { + if (!args[k]) + die(`missing --${k}`) + } +} + +// checksums lines are " " +function shaMap(checksumsPath) { + const map = new Map() + for (const line of readFileSync(checksumsPath, 'utf8').split('\n')) { + const m = line.match(/^([0-9a-f]{64})\s+(\S+)$/i) + if (m) + map.set(m[2], m[1]) + } + return map +} + +function emitManifest(args) { + requireArgs(args, ['channel', 'version', 'commit', 'build-date', 'base-url', 'checksums']) + validateVersionForChannel(args.version, args.channel) + const { release, compat } = loadPkg() + const shas = shaMap(args.checksums) + + const targetLines = release.targets.map((t) => { + const asset = assetName(release, args.version, t.id) + const sha = shas.get(asset) + if (!sha) + die(`no sha256 for ${asset} in ${args.checksums}`) + // one target per line: install-r2.sh grep/sed depends on this layout + return ` ${JSON.stringify(t.id)}: { "asset": ${JSON.stringify(asset)}, "sha256": ${JSON.stringify(sha)} }` + }).join(',\n') + + const head = { + schema: 1, + name: release.binName, + channel: args.channel, + version: args.version, + commit: args.commit, + buildDate: args['build-date'], + compat: { minDify: compat.minDify, maxDify: compat.maxDify }, + baseUrl: args['base-url'], + } + const headLines = Object.entries(head).map(([k, v]) => ` ${JSON.stringify(k)}: ${JSON.stringify(v)}`).join(',\n') + process.stdout.write(`{\n${headLines},\n "targets": {\n${targetLines}\n }\n}\n`) +} + +// Newline-delimited dir names of binaries that still exist in R2. Absent file = +// no reconciliation (caller could not list); empty file = no survivors. +function loadExistingDirs(path) { + if (!path || !existsSync(path)) + return null + const set = new Set() + for (const line of readFileSync(path, 'utf8').split('\n')) { + const d = line.trim() + if (d) + set.add(d) + } + return set +} + +function emitIndex(args) { + requireArgs(args, ['current', 'channel', 'version', 'commit', 'build-date']) + + // empty / "-" / missing = no ledger yet (first publish) + let current = { schema: 1, channel: args.channel, builds: [] } + if (args.current !== '-' && existsSync(args.current)) { + const raw = readFileSync(args.current, 'utf8').trim() + if (raw && raw !== '-') { + try { + current = JSON.parse(raw) + } + catch { + die(`current index at ${args.current} is not valid JSON`) + } + } + } + + const entry = { version: args.version, commit: args.commit, buildDate: args['build-date'], dir: args.version } + const kept = (current.builds ?? []).filter(b => b.version !== entry.version) + let builds = [entry, ...kept] + + // Reconcile to binaries that still exist in R2: lifecycle/TTL on the bin prefix + // is the only deletion mechanism, so the ledger never advertises a build whose + // binary is gone. The new build is always kept (just uploaded). No count cap. + const existing = loadExistingDirs(args['existing-dirs']) + if (existing) + builds = builds.filter(b => b.dir === entry.dir || existing.has(b.dir)) + + const index = { schema: 1, channel: args.channel, updated: args['build-date'], builds } + process.stdout.write(`${JSON.stringify(index, null, 2)}\n`) +} + +const [cmd, ...rest] = process.argv.slice(2) +const args = parseArgs(rest) +switch (cmd) { + case 'manifest': + emitManifest(args) + break + case 'index': + emitIndex(args) + break + default: + die(`unknown subcommand: ${cmd ?? '(none)'} (expected: manifest | index)`) +} diff --git a/cli/scripts/release-r2-edge.test.ts b/cli/scripts/release-r2-edge.test.ts new file mode 100644 index 0000000000..1514c6a28e --- /dev/null +++ b/cli/scripts/release-r2-edge.test.ts @@ -0,0 +1,210 @@ +import { execFileSync, spawnSync } from 'node:child_process' +import { mkdtempSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { describe, expect, it } from 'vitest' + +const SCRIPT = fileURLToPath(new URL('./release-r2-edge.mjs', import.meta.url)) + +function run(args: string[]): { code: number, stdout: string, stderr: string } { + try { + return { code: 0, stdout: execFileSync('node', [SCRIPT, ...args], { encoding: 'utf8' }), stderr: '' } + } + catch (e) { + const err = e as { status?: number, stdout?: string, stderr?: string } + return { code: err.status ?? 1, stdout: err.stdout ?? '', stderr: err.stderr ?? '' } + } +} + +// ---- manifest ---- + +function writeChecksums(version: string): string { + const dir = mkdtempSync(join(tmpdir(), 'difyctl-manifest-')) + const ids = ['linux-x64', 'linux-arm64', 'darwin-x64', 'darwin-arm64', 'windows-x64'] + const lines = ids.map((id, i) => { + const exe = id === 'windows-x64' ? '.exe' : '' + const sha = String(i).repeat(64) + return `${sha} difyctl-v${version}-${id}${exe}` + }) + const file = join(dir, `difyctl-v${version}-checksums.txt`) + writeFileSync(file, `${lines.join('\n')}\n`) + return file +} + +const VERSION = '0.1.0-edge.2fd7b82' +const BASE_URL = 'https://example.r2.dev/difyctl/edge/0.1.0-edge.2fd7b82' + +type ManifestJson = { + schema: number + name: string + channel: string + version: string + commit: string + buildDate: string + compat: { minDify: string, maxDify: string } + baseUrl: string + targets: Record +} + +type IndexBuild = { + version: string + commit: string + buildDate: string + dir: string +} + +type IndexJson = { + schema: number + channel: string + updated: string + builds: IndexBuild[] +} + +function buildManifest(version = VERSION): { code: number, json: ManifestJson, stdout: string, stderr: string } { + const checksums = writeChecksums(version) + const r = run(['manifest', '--channel', 'edge', '--version', version, '--commit', 'abc1234', '--build-date', '2026-06-14T12:00:00Z', '--base-url', BASE_URL, '--checksums', checksums]) + return { code: r.code, json: (r.code === 0 ? JSON.parse(r.stdout) : null) as ManifestJson, stdout: r.stdout, stderr: r.stderr } +} + +describe('release-r2-edge manifest', () => { + it('emits the core pointer fields', () => { + const { json } = buildManifest() + expect(json.schema).toBe(1) + expect(json.name).toBe('difyctl') + expect(json.channel).toBe('edge') + expect(json.version).toBe(VERSION) + expect(json.commit).toBe('abc1234') + expect(json.buildDate).toBe('2026-06-14T12:00:00Z') + expect(json.baseUrl).toBe(BASE_URL) + }) + + it('carries the compat window from package.json', () => { + const { json } = buildManifest() + expect(json.compat).toEqual({ minDify: '1.14.0', maxDify: '1.15.0' }) + }) + + it('lists all 5 targets with asset name + sha256 from the checksums file', () => { + const { json } = buildManifest() + expect(Object.keys(json.targets).sort()).toEqual( + ['darwin-arm64', 'darwin-x64', 'linux-arm64', 'linux-x64', 'windows-x64'], + ) + expect(json.targets['linux-x64'].asset).toBe(`difyctl-v${VERSION}-linux-x64`) + expect(json.targets['windows-x64'].asset).toBe(`difyctl-v${VERSION}-windows-x64.exe`) + expect(json.targets['linux-x64'].sha256).toMatch(/^\d{64}$/) + }) + + it('renders each target on a single line (installer greps it)', () => { + const { stdout } = buildManifest() + expect(stdout).toMatch(/^ {4}"linux-x64": \{ "asset": ".*", "sha256": ".*" \}/m) + }) + + it('rejects a version that does not match the channel form', () => { + const { code } = buildManifest('0.1.0-rc.1') + expect(code).not.toBe(0) + }) + + it('dies when a target sha is missing from the checksums file', () => { + const dir = mkdtempSync(join(tmpdir(), 'difyctl-manifest-')) + const file = join(dir, `difyctl-v${VERSION}-checksums.txt`) + writeFileSync(file, `${'0'.repeat(64)} difyctl-v${VERSION}-linux-x64\n`) // only 1 of 5 + const r = run(['manifest', '--channel', 'edge', '--version', VERSION, '--commit', 'abc1234', '--build-date', '2026-06-14T12:00:00Z', '--base-url', BASE_URL, '--checksums', file]) + expect(r.code).not.toBe(0) + }) + + it('rejects a malformed dropped-value argument (no silent misparse)', () => { + // --version has no value; --commit must NOT be swallowed as the version + const r = run(['manifest', '--channel', 'edge', '--version', '--commit', 'abc1234', '--build-date', '2026-06-14T12:00:00Z', '--base-url', 'https://x', '--checksums', '/nonexistent']) + expect(r.code).not.toBe(0) + }) +}) + +// ---- index ---- + +function runIndex(currentContent: string | null, build: Record, existingDirs?: string[]) { + let currentArg = '-' + if (currentContent !== null) { + const dir = mkdtempSync(join(tmpdir(), 'difyctl-index-')) + currentArg = join(dir, 'index.json') + writeFileSync(currentArg, currentContent) + } + const extra: string[] = [] + if (existingDirs !== undefined) { + const dir = mkdtempSync(join(tmpdir(), 'difyctl-existing-')) + const f = join(dir, 'existing.txt') + writeFileSync(f, `${existingDirs.join('\n')}\n`) + extra.push('--existing-dirs', f) + } + const r = spawnSync('node', [SCRIPT, 'index', '--current', currentArg, '--channel', 'edge', '--version', build.version, '--commit', build.commit, '--build-date', build.buildDate, ...extra], { encoding: 'utf8' }) + return { + code: r.status ?? 1, + index: (r.status === 0 ? JSON.parse(r.stdout) : null) as IndexJson, + } +} + +const B1 = { version: '0.1.0-edge.aaaaaaa', commit: 'aaaaaaa', buildDate: '2026-06-14T09:00:00Z' } +const B2 = { version: '0.1.0-edge.bbbbbbb', commit: 'bbbbbbb', buildDate: '2026-06-14T10:00:00Z' } + +describe('release-r2-edge index', () => { + it('creates a fresh index from a missing current (arg "-")', () => { + const { index } = runIndex(null, B1) + expect(index.schema).toBe(1) + expect(index.channel).toBe('edge') + expect(index.builds).toHaveLength(1) + expect(index.builds[0]).toMatchObject({ version: B1.version, commit: B1.commit, dir: B1.version }) + }) + + it('treats an empty current file as fresh (first publish, curl wrote nothing)', () => { + const { code, index } = runIndex('', B1) + expect(code).toBe(0) + expect(index.builds).toHaveLength(1) + }) + + it('treats a "-"-content current file as fresh (curl 404 fallback)', () => { + const { code, index } = runIndex('-\n', B1) + expect(code).toBe(0) + expect(index.builds).toHaveLength(1) + }) + + it('prepends the new build (publish order; newest at [0])', () => { + const current = JSON.stringify({ schema: 1, channel: 'edge', builds: [{ version: B1.version, commit: B1.commit, buildDate: B1.buildDate, dir: B1.version }] }) + const { index } = runIndex(current, B2) + expect(index.builds.map(b => b.version)).toEqual([B2.version, B1.version]) + }) + + it('dedups a re-cut of the same version (no duplicate, moves to top)', () => { + const current = JSON.stringify({ schema: 1, channel: 'edge', builds: [ + { version: B2.version, commit: B2.commit, buildDate: B2.buildDate, dir: B2.version }, + { version: B1.version, commit: B1.commit, buildDate: B1.buildDate, dir: B1.version }, + ] }) + const { index } = runIndex(current, B1) // re-cut B1 + expect(index.builds.map(b => b.version)).toEqual([B1.version, B2.version]) + }) + + it('reconciles to surviving binary dirs (drops a build whose binary expired)', () => { + const current = JSON.stringify({ schema: 1, channel: 'edge', builds: [ + { version: B1.version, commit: B1.commit, buildDate: B1.buildDate, dir: B1.version }, + ] }) + // B1's binary is gone (not in existing); the new B2 is always kept. + const { index } = runIndex(current, B2, [B2.version]) + expect(index.builds.map(b => b.version)).toEqual([B2.version]) + }) + + it('keeps the new build even when it is absent from the existing-dirs list', () => { + const { index } = runIndex(null, B1, []) // empty survivors, fresh ledger + expect(index.builds.map(b => b.version)).toEqual([B1.version]) + }) + + it('does not reconcile when no --existing-dirs is given (list unavailable)', () => { + const current = JSON.stringify({ schema: 1, channel: 'edge', builds: [ + { version: B1.version, commit: B1.commit, buildDate: B1.buildDate, dir: B1.version }, + ] }) + const { index } = runIndex(current, B2) // no existing-dirs → keep all + expect(index.builds.map(b => b.version)).toEqual([B2.version, B1.version]) + }) + + it('dies on a non-empty current file that is not valid JSON', () => { + const { code } = runIndex('{not json', B1) + expect(code).not.toBe(0) + }) +}) diff --git a/cli/scripts/release-r2-publish.sh b/cli/scripts/release-r2-publish.sh new file mode 100755 index 0000000000..4b41150401 --- /dev/null +++ b/cli/scripts/release-r2-publish.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# release-r2-publish.sh — direct-publish difyctl binaries + manifest + index to R2. +# Strict order so the pointer never references missing bytes: +# sync binaries -> HEAD-verify -> list survivors -> put index.json -> put manifest.json. +# Binaries live under the bin prefix and expire via an R2 lifecycle (TTL) rule on +# that prefix; the ledger is reconciled to surviving dirs, not pruned here. The +# pointer JSONs (manifest.json/index.json) live under the (non-expiring) prefix. +# Installers (install-r2.sh/.ps1) are served from the GitHub repo, not R2. +# Usage: release-r2-publish.sh +# Env: DIFYCTL_R2_S3_ENDPOINT DIFYCTL_R2_BUCKET DIFYCTL_R2_PUBLIC_BASE (+ AWS creds, AWS_REQUEST_CHECKSUM_CALCULATION). +# DIFYCTL_R2_PREFIX (default difyctl) key root for pointer JSONs +# DIFYCTL_R2_BIN_PREFIX (default /bin) key root for binaries (TTL target) +set -euo pipefail + +_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DIST_DIR="${DIST_DIR:-${_dir}/../dist/bin}" + +aws_s3() { aws --endpoint-url "$DIFYCTL_R2_S3_ENDPOINT" "$@"; } + +publish_main() { + local channel="$1" version="$2" + local prefix="${DIFYCTL_R2_PREFIX:-difyctl}" + local bin_prefix="${DIFYCTL_R2_BIN_PREFIX:-${prefix}/bin}" + local key="${bin_prefix}/${channel}/${version}" + local pointer_prefix="${prefix}/${channel}" + local base_url="${DIFYCTL_R2_PUBLIC_BASE}/${key}" + + local current="" new_index="" existing_dirs="" manifest="" + trap 'rm -f "$current" "$new_index" "$existing_dirs" "$manifest"' RETURN + + # 1. binaries (+ checksums) — immutable + aws_s3 s3 sync "$DIST_DIR" "s3://${DIFYCTL_R2_BUCKET}/${key}/" \ + --content-type application/octet-stream \ + --cache-control "public, max-age=31536000, immutable" + + # 2. HEAD-verify each expected target asset exists on R2. Capture targets into + # a variable first; a process-substitution producer failure bypasses set -e. + local targets + targets="$(node "${_dir}/release-naming.mjs" targets)" + [ -n "$targets" ] || { echo "release-r2-publish: no release targets resolved" >&2; exit 1; } + local verified=0 _bun id _exe asset + while IFS=$'\t' read -r _bun id _exe; do + asset="$(node "${_dir}/release-naming.mjs" asset "$version" "$id")" + aws_s3 s3api head-object --bucket "$DIFYCTL_R2_BUCKET" --key "${key}/${asset}" >/dev/null + verified=$((verified + 1)) + done <<< "$targets" + [ "$verified" -gt 0 ] || { echo "release-r2-publish: verified zero targets" >&2; exit 1; } + + # 3. survivors: binary dirs still present under the bin prefix (lifecycle/TTL may + # have deleted old ones). aws CLI auto-paginates list-objects-v2, so --query + # aggregates CommonPrefixes across all pages. On a list failure, skip + # reconciliation (keep the ledger as-is) rather than wrongly drop builds. + existing_dirs="$(mktemp)" + local listed=0 raw + raw="$(mktemp)" + if aws_s3 s3api list-objects-v2 --bucket "$DIFYCTL_R2_BUCKET" \ + --prefix "${bin_prefix}/${channel}/" --delimiter / \ + --query 'CommonPrefixes[].Prefix' --output text > "$raw" 2>/dev/null; then + # text rows are "///"; emit just . "None" = empty. + tr '\t' '\n' < "$raw" | awk -F/ 'NF>1 && $0 != "None" { print $(NF-1) }' > "$existing_dirs" + listed=1 + fi + rm -f "$raw" + + # 4. index.json: fetch current, prepend, reconcile to survivors. + current="$(mktemp)" + curl -fsSL "${DIFYCTL_R2_PUBLIC_BASE}/${pointer_prefix}/index.json" -o "$current" 2>/dev/null || echo '-' > "$current" + new_index="$(mktemp)" + if [ "$listed" = "1" ]; then + emit_index "$current" "$channel" "$version" --existing-dirs "$existing_dirs" >"$new_index" + else + emit_index "$current" "$channel" "$version" >"$new_index" + fi + aws_s3 s3 cp "$new_index" "s3://${DIFYCTL_R2_BUCKET}/${pointer_prefix}/index.json" \ + --content-type application/json --cache-control "public, max-age=60, must-revalidate" + + # 5. manifest.json — pointer; written last so it never references missing binaries + manifest="$(mktemp)" + node "${_dir}/release-r2-edge.mjs" manifest --channel "$channel" --version "$version" \ + --commit "${DIFYCTL_COMMIT:-unknown}" --build-date "${DIFYCTL_BUILD_DATE:-unknown}" \ + --base-url "$base_url" --checksums "${DIST_DIR}/difyctl-v${version}-checksums.txt" >"$manifest" + aws_s3 s3 cp "$manifest" "s3://${DIFYCTL_R2_BUCKET}/${pointer_prefix}/manifest.json" \ + --content-type application/json --cache-control "public, max-age=60, must-revalidate" +} + +# emit_index [extra args...] — wrapper so the +# survivor flag can be omitted without tripping `set -u` on empty arrays (bash 3.2). +emit_index() { + local current="$1" channel="$2" version="$3"; shift 3 + node "${_dir}/release-r2-edge.mjs" index --current "$current" --channel "$channel" \ + --version "$version" --commit "${DIFYCTL_COMMIT:-unknown}" --build-date "${DIFYCTL_BUILD_DATE:-unknown}" "$@" +} + +if [ "${RELEASE_PUBLISH_LIB:-0}" != "1" ]; then + [ "$#" -eq 2 ] || { echo "usage: release-r2-publish.sh " >&2; exit 2; } + publish_main "$1" "$2" +fi diff --git a/cli/scripts/release-r2-publish.test.ts b/cli/scripts/release-r2-publish.test.ts new file mode 100644 index 0000000000..0b99622c3d --- /dev/null +++ b/cli/scripts/release-r2-publish.test.ts @@ -0,0 +1,87 @@ +import { spawnSync } from 'node:child_process' +import { fileURLToPath } from 'node:url' +import { describe, expect, it } from 'vitest' + +const SCRIPT = fileURLToPath(new URL('./release-r2-publish.sh', import.meta.url)) + +// Stub `aws` + `curl` + `node` as shell functions that just log action verbs to +// $ORDER_LOG, then run the publish `main` and assert the order of operations. +function runPublish(): { code: number, order: string[], stderr: string } { + const stub = [ + 'ORDER_LOG="$(mktemp)"', + 'aws() {', + ' case "$*" in', + ' *"list-objects-v2"*) echo list-survivors >>"$ORDER_LOG" ;;', + ' *" cp "*"/index.json"*) echo put-index >>"$ORDER_LOG" ;;', + ' *" cp "*"/manifest.json"*) echo put-manifest >>"$ORDER_LOG" ;;', + ' *" sync "*) echo sync-binaries >>"$ORDER_LOG" ;;', + ' *"head-object"*) echo head-verify >>"$ORDER_LOG" ;;', + ' *) : ;;', + ' esac', + '}', + 'curl() { echo "{}"; }', + 'node() {', + ' case "$*" in', + ' *release-naming.mjs*targets*)', + ' printf \'bun-linux-x64\\tlinux-x64\\t0\\nbun-linux-arm64\\tlinux-arm64\\t0\\nbun-darwin-x64\\tdarwin-x64\\t0\\nbun-darwin-arm64\\tdarwin-arm64\\t0\\nbun-windows-x64\\twindows-x64\\t1\\n\' ;;', + ' *release-naming.mjs*\' asset \'*) printf \'difyctl-vX\\n\' ;;', + ' *release-r2-edge.mjs*\' index \'*) echo \'{}\' ;;', + ' *release-r2-edge.mjs*\' manifest \'*) echo \'{}\' ;;', + ' *) : ;;', + ' esac', + '}', + ].join('\n') + const program = [ + stub, + `. "${SCRIPT}"`, + 'publish_main edge 0.1.0-edge.2fd7b82', + 'cat "$ORDER_LOG"', + ].join('\n') + // bash, NOT sh: the script uses BASH_SOURCE + process substitution. + const r = spawnSync('bash', ['-c', program], { + encoding: 'utf8', + env: { + ...process.env, + RELEASE_PUBLISH_LIB: '1', + DIFYCTL_R2_S3_ENDPOINT: 'https://endpoint.example', + DIFYCTL_R2_BUCKET: 'cli-dev', + DIFYCTL_R2_PUBLIC_BASE: 'https://pub.example.r2.dev', + DIST_DIR: '/tmp', + }, + }) + return { code: r.status ?? 1, order: (r.stdout ?? '').trim().split('\n').filter(Boolean), stderr: r.stderr ?? '' } +} + +describe('release-r2-publish order', () => { + it('uploads binaries, verifies, lists survivors, then index, then manifest', () => { + const { code, order } = runPublish() + expect(code).toBe(0) + expect(order.indexOf('sync-binaries')).toBeLessThan(order.indexOf('head-verify')) + expect(order.indexOf('head-verify')).toBeLessThan(order.indexOf('list-survivors')) + expect(order.indexOf('list-survivors')).toBeLessThan(order.indexOf('put-index')) + expect(order.indexOf('put-index')).toBeLessThan(order.indexOf('put-manifest')) + // pointer is never pruned here — deletion is owned by the R2 lifecycle rule + expect(order).not.toContain('prune') + }) + + it('exits non-zero when no targets resolve (head-verify safety gate)', () => { + const stub = [ + 'aws() { :; }', + 'curl() { echo "{}"; }', + 'node() { case "$*" in *release-naming.mjs*targets*) : ;; *) echo "{}" ;; esac; }', + ].join('\n') + const program = [stub, `. "${SCRIPT}"`, 'publish_main edge 0.1.0-edge.2fd7b82'].join('\n') + const r = spawnSync('bash', ['-c', program], { + encoding: 'utf8', + env: { + ...process.env, + RELEASE_PUBLISH_LIB: '1', + DIFYCTL_R2_S3_ENDPOINT: 'https://endpoint.example', + DIFYCTL_R2_BUCKET: 'cli-dev', + DIFYCTL_R2_PUBLIC_BASE: 'https://pub.example.r2.dev', + DIST_DIR: '/tmp', + }, + }) + expect(r.status).not.toBe(0) + }) +}) diff --git a/cli/src/version/info.ts b/cli/src/version/info.ts index 23b8624b34..e8d38180fc 100644 --- a/cli/src/version/info.ts +++ b/cli/src/version/info.ts @@ -1,7 +1,7 @@ import { arch, platform } from '@/sys/index' import { compatString } from './compat' -export type Channel = 'dev' | 'rc' | 'stable' +export type Channel = 'dev' | 'edge' | 'rc' | 'stable' export type VersionInfo = { version: string