diff --git a/Dockerfile b/Dockerfile index 54bded77..85f14b67 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,17 @@ +# SqlPackage, for the Azure SQL Database adapter. +# +# Built in its own stage so the .NET SDK never reaches the runtime image. The tool +# is published as portable IL under tools//any, so this produces the correct +# architecture automatically: buildx runs this stage once per target platform. +# Verified on linux/arm64 as well as linux/amd64 - Microsoft's standalone zip is +# x64-only, which is why the dotnet tool is used instead of the download. +FROM mcr.microsoft.com/dotnet/sdk:10.0 AS sqlpackage-build +RUN dotnet tool install --tool-path /tmp/sqlpkg microsoft.sqlpackage && \ + PAYLOAD="$(find /tmp/sqlpkg/.store -type d -path '*/tools/net10.0/any' | head -1)" && \ + test -n "$PAYLOAD" && \ + mkdir -p /opt/sqlpackage && \ + cp -a "$PAYLOAD"/. /opt/sqlpackage/ + # Base Image: Node.js 24 on Debian Slim (bookworm) FROM node:24-slim AS base @@ -98,6 +112,26 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ldconfig && \ rm -rf /tmp/firebird.tar.gz /tmp/firebird-extract +# Step 6: SqlPackage runtime, for the Azure SQL Database adapter. +# +# Only the .NET runtime, never the SDK - the tool itself came from the stage above. +# libicu is a hard dependency: without it .NET aborts at startup with a globalization +# error that says nothing about the missing package. +# +# The wrapper exists because the adapter resolves the binary with host.which("sqlpackage"), +# and the apphost shim from a --tool-path install is not on PATH here. +RUN apt-get update && apt-get install -y --no-install-recommends libicu72 && \ + rm -rf /var/lib/apt/lists/* && \ + curl -fsSL https://dot.net/v1/dotnet-install.sh -o /tmp/dotnet-install.sh && \ + bash /tmp/dotnet-install.sh --channel 10.0 --runtime dotnet --install-dir /usr/share/dotnet --no-path && \ + rm /tmp/dotnet-install.sh + +COPY --from=sqlpackage-build /opt/sqlpackage /opt/sqlpackage + +RUN printf '#!/bin/sh\nexec /usr/share/dotnet/dotnet /opt/sqlpackage/sqlpackage.dll "$@"\n' > /usr/local/bin/sqlpackage && \ + chmod +x /usr/local/bin/sqlpackage && \ + sqlpackage /version + # Enable corepack for pnpm support and symlink PostgreSQL 18 binaries # On Debian with PGDG, pg binaries live under /usr/lib/postgresql/18/bin/ RUN corepack enable && corepack prepare pnpm@10.29.3 --activate && \ diff --git a/README.md b/README.md index 7e0d5b7c..1add5fd9 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ Redis Valkey MSSQL + Azure SQL Database Firebird
License @@ -61,7 +62,7 @@ That promise shapes the architecture: incremental backups store whole changed fi ### ๐Ÿ—„๏ธ Database Backup -- **9 Database Engines** - MySQL, MariaDB, PostgreSQL, MongoDB, SQLite, Redis, Valkey, Microsoft SQL Server, and Firebird (beta) +- **10 Database Engines** - MySQL, MariaDB, PostgreSQL, MongoDB, SQLite, Redis, Valkey, Microsoft SQL Server, Azure SQL Database (beta), and Firebird (beta) - **Selective Database Backup** - Choose exactly which databases to back up per job instead of creating separate sources for each database - **Multi-Database Jobs** - Back up multiple databases from a single source in one job with a unified TAR archive format - **AES-256-GCM Encryption** - Encrypt backups with managed Encryption Profiles, key rotation, and downloadable Recovery Kits for offline decryption @@ -185,6 +186,7 @@ Open [https://localhost:3000](https://localhost:3000) and create your admin acco | Valkey | 7.2+ | Direct, SSH | Guided | | SQLite | 3.x | Local, SSH | Yes | | Microsoft SQL Server | 2017, 2019, 2022, Azure SQL Edge | Direct, SSH | Yes | +| Azure SQL Database (beta) | Single database, elastic pool | Direct | Yes (drops the target first) | | Firebird (beta) | 3.x, 4.x, 5.x | Direct, SSH | Yes (pre-configured aliases) | ## ๐Ÿ“ Directory Sources diff --git a/docker-compose.yml b/docker-compose.yml index 39e8c299..1b4a267c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,4 +18,5 @@ services: # - OIDC_AUTO_REDIRECT=authentik-742 # Optional: SSO provider ID to redirect to instead of the login page volumes: - ./data:/data # All persistent data (db, storage, certs) - - ./backups:/backups # Optional: used for local backups \ No newline at end of file + - ./backups:/backups # Optional: used for local backups + # - ./tmp:/tmp # Recommended: staging space for running backups, keeps them off the Docker disk \ No newline at end of file diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index ac5ffde0..c3941784 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -110,6 +110,7 @@ export default defineConfig({ { text: 'Valkey', link: '/user-guide/sources/valkey' }, { text: 'SQLite', link: '/user-guide/sources/sqlite' }, { text: 'Microsoft SQL Server', link: '/user-guide/sources/mssql' }, + { text: 'Azure SQL Database', link: '/user-guide/sources/azure-sql' }, { text: 'Firebird', link: '/user-guide/sources/firebird' } ] }, diff --git a/docs/changelog.md b/docs/changelog.md index 787ddfe8..eed965e0 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,64 @@ All notable changes to DBackup are documented here. +## vNEXT +*Release: In Progress* + +> โš ๏ธ **Breaking:** Retention now decides how old a backup is from the creation time DBackup recorded in its `.meta.json`, not from the file's modification time on the destination. Where the two still agree, which is the normal case, the same backups are kept as before and nothing needs doing. Where they were pulled apart, by moving a destination or copying it without preserving timestamps, retention keeps a different set from the next run onwards. That is the intended fix, because a reset modification time collapses the whole history into a single bucket and costs almost all of it, but it does mean the first run after updating can delete backups the run before it kept. Open the retention step of that first run and look for lines naming a backup whose recorded time and modification time disagree. Lock anything you cannot lose before a destination is moved. + +### โœจ Features + +- **retention**: Smart (GFS) policies can now keep an hourly tier next to daily, weekly, monthly and yearly. The field stays hidden behind **Add hourly tier** until it is needed, so existing policies keep their behaviour unchanged. +- **azure-sql**: New **Azure SQL Database** source type in beta, backing up through a BACPAC export. + +### ๐Ÿ› Bug Fixes + +- **settings**: The System Timezone now accepts renamed zones such as **Asia/Kolkata** and **Europe/Kyiv**, which were refused with **Invalid IANA timezone** in browsers that offer the modern name. Around 140 zones were affected, and the picker now also keeps the stored zone selectable in browsers that only know its legacy name. ([#147](https://github.com/Skyfay/DBackup/issues/147)) +- **retention**: A policy whose mode carries no settings, such as **Smart** with no tiers stored, now keeps every backup instead of deleting all of them. Only configurations written through the API could reach this state. +- **mssql**: Azure SQL Database and Azure SQL Managed Instance are now refused up front with a message naming the product, instead of connecting successfully and then failing partway through a backup with a raw T-SQL error. Both are also named correctly in the connection test, where they previously showed as **SQL**. +- **mssql**: The Database Explorer now lists databases on servers that do not expose **sys.master_files**, showing names and table counts without sizes rather than failing the whole page with **Connection Failed**. +- **mssql**: Restoring a database under a different name now places its files in the instance's own default data and log directories, which is what makes such a restore work against a SQL Server running on Windows. A database holding more than one data file no longer has all of them moved onto the same file. ([#148](https://github.com/Skyfay/DBackup/issues/148)) +- **s3**: Listing a bucket now returns every object instead of stopping at the first 1000. Because the cut fell alphabetically and backup names carry timestamps, the newest backups were the ones missing from retention, integrity checks, the destination browser and the dashboard. +- **s3**: Empty files now appear in listings on S3, Cloudflare R2, Hetzner Object Storage and S3-compatible providers. They were dropped along with folder markers, which left them out of directory backups and made them read as deleted everywhere a listing decides what still exists. + +### ๐Ÿ”„ Changed + +- **retention**: Backups are now sorted into their hourly, daily, weekly, monthly and yearly buckets by the creation time DBackup recorded when it wrote them. The file's modification time on the destination is only used for backups that have no recorded time. + +### ๐ŸŽจ Improvements + +- **retention**: Retention policies are now validated before they are saved. A negative, fractional or non numeric tier is rejected instead of stored. +- **retention**: Reading backup metadata at the end of a job now runs up to 8 requests at once on S3, WebDAV, Dropbox, Google Drive, OneDrive and local destinations, and skips backups the listing already shows have no metadata. FTP, SMB, SFTP and rsync stay sequential because each read there costs a connection or a process. +- **retention**: The run log now names any backup whose recorded creation time disagrees with the destination's modification time by more than an hour, and reports how many backups supplied their own time. +- **s3**: Backups now upload to S3, Cloudflare R2, Hetzner Object Storage and S3-compatible providers in 8 parallel parts of 8 MB instead of the AWS SDK's 4 parts of 5 MB. A 1.29 GB archive to R2 moved at 27 MB/s before the change while the same run read and hashed it locally at over 400 MB/s. +- **s3**: New **Parallel Upload Parts** setting on every S3 backup destination sets how many parts upload at once and how large each one may be, up to 32 parts of 64 MB. The form shows how much memory the chosen combination uses per upload. +- **s3**: Part size now adapts to the backup being uploaded, never above the configured maximum. A part size too large for a given archive used to leave connections with nothing to upload, which cost a 1.39 GB backup to Cloudflare R2 a third of its throughput at 32 parts of 64 MB. +- **s3**: The run log now records upload throughput and the part size actually used. Throughput was only ever shown in the live progress detail, which is gone once the run ends. +- **s3**: Backing up a directory from an S3 destination now reports progress while the listing runs and stops within one request when the job is cancelled. Both previously waited for the entire listing to finish. + +### ๐Ÿ“ Documentation + +- **installation**: The installation guide now recommends mounting `/tmp` so a running backup is staged outside the Docker disk, and the compose and run examples carry the volume. The file backup guide and the environment reference explain the same thing where disk space comes up. ([#145](https://github.com/Skyfay/DBackup/issues/145)) +- **developer-guide**: The setup guide now points at the platform setup scripts instead of listing a shorter set of packages beside them. It also warns against installing `libpq` for PostgreSQL, whose `pg_dump` is built without LZ4 and ZSTD and breaks native compression. +- **mssql**: The guide now covers SQL Server on Windows, from the form the backup path has to take to setting up SSH mode against the Windows OpenSSH server. An SMB share is documented as the fallback where that server is unavailable. ([#148](https://github.com/Skyfay/DBackup/issues/148)) + +### ๐Ÿงช Tests + +- **tests**: The adapter transport lint guard no longer fails under load. It imports the entire adapter registry and was running against the default 5 second limit, which every new adapter moved a little closer to the edge. + +### ๐Ÿ”ง CI/CD + +- **docker**: The image now ships SqlPackage and the .NET runtime, which the Azure SQL Database source needs. This adds roughly 270 MB on both **linux/amd64** and **linux/arm64**. +- **scripts**: The macOS and Debian development setup scripts now install SqlPackage, which the Azure SQL Database source needs. On macOS it lands in the Homebrew prefix and is wrapped so it needs neither a `PATH` entry nor a `DOTNET_ROOT` variable. + +### ๐Ÿณ Docker + +- **Image**: `skyfay/dbackup:vNEXT` +- **Also tagged as**: `latest`, `vNEXT` +- **CI Image**: `skyfay/dbackup:ci` +- **Platforms**: linux/amd64, linux/arm64 + + ## v3.2.0 - Docker Volumes Backup, SSH Key Generation, MongoDB Atlas Support, and Bug Fixes *Released: Aug 8, 2026* diff --git a/docs/developer-guide/adapters/database.md b/docs/developer-guide/adapters/database.md index 36c2b62e..513f5746 100644 --- a/docs/developer-guide/adapters/database.md +++ b/docs/developer-guide/adapters/database.md @@ -12,6 +12,7 @@ Database adapters handle the dump and restore operations for different database | MongoDB | `mongodb` | `mongodump`, `mongorestore` | โœ… | `.archive` | | SQLite | `sqlite` | None (file copy) | โœ… | `.db` | | MSSQL | `mssql` | None (TDS protocol) | โœ… (TDS tunnelled) | `.bak` | +| Azure SQL Database | `azure-sql` | `sqlpackage` | โŒ (public PaaS endpoint) | `.bacpac` | | Redis | `redis` | `redis-cli` | โœ… | `.rdb` | | Firebird | `firebird` | `gbak`, `isql` | โœ… | `.fbk` | @@ -28,6 +29,7 @@ getBackupFileExtension("redis"); // "rdb" getBackupFileExtension("mongodb"); // "archive" getBackupFileExtension("sqlite"); // "db" getBackupFileExtension("mssql"); // "bak" +getBackupFileExtension("azure-sql"); // "bacpac" getBackupFileExtension("firebird"); // "fbk" ``` @@ -38,6 +40,7 @@ getBackupFileExtension("firebird"); // "fbk" | MySQL/MariaDB | `.sql` | Standard SQL dump format | | PostgreSQL | `.sql` | SQL dump (or `.dump` for custom format) | | MSSQL | `.bak` | Native SQL Server backup format | +| Azure SQL Database | `.bacpac` | SqlPackage data-tier application export, a ZIP so it is never recompressed | | MongoDB | `.archive` | mongodump `--archive` format | | Redis | `.rdb` | Redis Database snapshot format | | SQLite | `.db` | Direct database file copy | @@ -129,6 +132,7 @@ Each database adapter can optionally return size and table count information. Th | **PostgreSQL** | `pg_database_size(datname)` | `COUNT(*)` from `information_schema.tables` (excl. system schemas) | | **MongoDB** | Native `sizeOnDisk` from `listDatabases` command | `listCollections().length` per database | | **MSSQL** | `sys.master_files` (`SUM(size) * 8 * 1024`) | `COUNT(*)` from `INFORMATION_SCHEMA.TABLES` | +| **Azure SQL Database** | `sys.database_files` (`SUM(size) * 8 * 1024`, data files only), one connection per database | `COUNT(*)` from `sys.tables`, same connection | | **SQLite** | Not supported | Not supported | | **Redis** | Not supported | Not supported | diff --git a/docs/developer-guide/advanced/retention.md b/docs/developer-guide/advanced/retention.md index e1caf58d..1bc759f4 100644 --- a/docs/developer-guide/advanced/retention.md +++ b/docs/developer-guide/advanced/retention.md @@ -17,8 +17,8 @@ DBackup supports three retention modes: The GFS algorithm keeps backups at decreasing frequencies as they age: ``` -Today โ†โ”€โ”€โ”€โ”€ Daily โ”€โ”€โ”€โ”€โ†’ Weekly โ”€โ”€โ”€โ”€โ†’ Monthly โ”€โ”€โ”€โ”€โ†’ Yearly - โ†โ”€โ”€โ”€ 7 days โ”€โ”€โ”€โ†’ 4 weeks โ”€โ”€โ”€โ†’ 12 months โ”€โ”€โ†’ โˆž +Now โ†โ”€ Hourly โ”€โ†’ Daily โ”€โ”€โ”€โ”€โ†’ Weekly โ”€โ”€โ”€โ”€โ†’ Monthly โ”€โ”€โ”€โ”€โ†’ Yearly + โ†โ”€ 24 hours โ”€โ†’ 7 days โ”€โ†’ 4 weeks โ”€โ”€โ”€โ†’ 12 months โ”€โ”€โ†’ โˆž ``` ### Example Configuration @@ -27,26 +27,64 @@ Today โ†โ”€โ”€โ”€โ”€ Daily โ”€โ”€โ”€โ”€โ†’ Weekly โ”€โ”€โ”€โ”€โ†’ Monthly โ”€ { "mode": "SMART", "smart": { - "daily": 7, // Keep last 7 daily backups - "weekly": 4, // Keep last 4 weekly backups - "monthly": 6, // Keep last 6 monthly backups - "yearly": 2 // Keep last 2 yearly backups + "hourly": 24, // Keep last 24 hourly backups (optional) + "daily": 7, // Keep 7 further daily backups + "weekly": 4, // Keep 4 further weekly backups + "monthly": 6, // Keep 6 further monthly backups + "yearly": 2 // Keep 2 further yearly backups } } ``` ### How Selection Works -1. **Daily**: Most recent backup from each of the last N days -2. **Weekly**: Most recent backup from each of the last N weeks -3. **Monthly**: Most recent backup from each of the last N months -4. **Yearly**: Most recent backup from each of the last N years +Tiers run finest first over the file list sorted newest to oldest. Each tier keeps the first file it sees in a bucket it has not covered yet, which is the newest backup of that bucket. -A single backup can satisfy multiple buckets. For example, January 1st's backup could be: -- Today's daily backup -- This week's weekly backup -- This month's monthly backup -- This year's yearly backup +1. **Hourly**: most recent backup from each of the last N hours +2. **Daily**: most recent backup from each of the next N days +3. **Weekly**: most recent backup from each of the next N weeks +4. **Monthly**: most recent backup from each of the next N months +5. **Yearly**: most recent backup from each of the next N years + +**The tiers are additive, not overlapping.** `applyTier` seeds its bucket set from everything earlier tiers already kept and counts only its own additions against its limit. `daily: 7` therefore means seven days beyond what the hourly tier covers, and the total kept is the sum of the tiers. restic and borg evaluate the same numbers as a union, so an identical config keeps fewer backups there. + +Bucket keys are built with `formatInTimeZone` against the `system.timezone` setting, so a day boundary is local midnight. The hourly key is `yyyy-MM-dd-HH`, which collapses the repeated hour of a daylight saving change into one bucket once a year. + +### Which time a file is bucketed by + +`effectiveBackupTime()` in `src/lib/core/backup-files.ts` is the single rule, used by both the sort and the bucket key: + +```typescript +file.backupTimestamp ?? file.lastModified +``` + +`lastModified` is whatever the adapter's `list()` reports, so an S3 `LastModified`, an SFTP `modifyTime`, a local `stats.mtime`. It is not a reliable statement about when the backup was taken. Copying a destination without preserving timestamps, moving it between servers or restoring the backup directory itself resets every mtime to now, at which point the whole history lands in one bucket and a single representative survives the next pass. + +`backupTimestamp` comes from `timestamp` in the backup's `.meta.json`, written at upload in `03-upload.ts`, and survives all of that. It is left unset when the sidecar is missing, unreadable or carries an unparsable date, so the mtime stays the fallback rather than the rule. The filename is never parsed, even though the naming template puts a date in it. + +`05-retention.ts` reports how many backups supplied their own time and warns by name for each one whose two times differ by more than `TIMESTAMP_DRIFT_WARNING_MS`. + +### Reading the sidecars + +`loadBackupSidecars()` in `src/lib/runner/steps/retention-sidecars.ts` annotates the listed files with `locked`, `chainId` and `backupTimestamp`. It runs once per destination at the end of every successful job, over every backup present, so its round trip count is the dominant cost of the whole step. + +Two things keep that bounded: + +- **Sidecars absent from the listing are never requested.** `list()` returns sidecars, they are only filtered out afterwards, so their presence can be answered from the listing for free. The optimisation disables itself when a listing contains no sidecars at all, otherwise an adapter that filters them would silently lose lock and chain detection. +- **Reads run in batches of `adapter.readConcurrency`.** Unset means serial, which is what every adapter did before the field existed. Only adapters whose `read()` is a stateless HTTP request or a local file access declare `STATELESS_READ_CONCURRENCY`, currently S3, WebDAV, Dropbox, Google Drive, OneDrive and Local. + +FTP, SMB, SFTP and rsync deliberately declare nothing. FTP dials a control connection per `read()` and its own upload path runs at `limit: concurrency ?? 1` for exactly that reason, SMB spawns an `smbclient` process per call, and the two SSH-based adapters already gate themselves at four channels. On those the server's connection count is what breaks first, not the bandwidth. + +### Tier limits and backwards compatibility + +`hourly` is optional on `SmartRetentionPolicy` because every policy written before the tier existed has no value for it. Two places turn that into a disabled tier: + +- `applySmartPolicy` destructures with `const { hourly = 0, ... }` +- `applyTier` guards with `if (!limit || limit <= 0) return;` + +The guard cannot be written as `limit <= 0` alone. `undefined <= 0` evaluates to `false` in JavaScript, so the tier would run with `keptInTier >= undefined` never true, keep one backup per bucket for the whole history, and silently stop deleting anything. + +`calculateRetention` also returns the full keep list when a mode carries no usable settings. Without that branch nothing marks a file as kept and every unlocked backup on the destination ends up in the delete list. ## Data Model @@ -71,6 +109,7 @@ export interface RetentionConfiguration { keepCount: number; }; smart?: { + hourly?: number; // optional, absent counts as 0 daily: number; weekly: number; monthly: number; @@ -79,6 +118,8 @@ export interface RetentionConfiguration { } ``` +`RetentionConfigurationSchema` in the same file validates a config before `retention-policy-service.ts` stores it. Tier limits are coerced to non negative integers, so a value written through the API cannot reach the bucketing logic malformed. + ## RetentionService Implementation The core logic lives in `src/services/retention-service.ts`: diff --git a/docs/developer-guide/reference/archive-format.md b/docs/developer-guide/reference/archive-format.md index 2b478d4c..a98af8c0 100644 --- a/docs/developer-guide/reference/archive-format.md +++ b/docs/developer-guide/reference/archive-format.md @@ -71,8 +71,8 @@ Encrypted archives use opaque names on purpose. TAR headers are not encrypted, s paths there would publish the file listing next to the encrypted data and make the sealed index pointless. -`` is `sql`, `dump` (PostgreSQL custom format), `archive` (MongoDB), `bak` (MSSQL) or -`fbk` (Firebird). +`` is `sql`, `dump` (PostgreSQL custom format), `archive` (MongoDB), `bak` (MSSQL), +`bacpac` (Azure SQL Database) or `fbk` (Firebird). ## manifest.json diff --git a/docs/developer-guide/reference/environment.md b/docs/developer-guide/reference/environment.md index 0694e199..9e09c92e 100644 --- a/docs/developer-guide/reference/environment.md +++ b/docs/developer-guide/reference/environment.md @@ -42,7 +42,7 @@ Complete reference for all environment variables in DBackup. - **PORT** changes the internal port. When using custom ports, set both `PORT` and update your port mapping accordingly - **DATABASE_URL** has a sensible default and typically doesn't need to be set - **SQLITE_WAL_MODE** enables [WAL (Write-Ahead Logging)](https://www.sqlite.org/wal.html) mode by default so readers and the writer don't block each other. Set to `false` only if `/data` is on a filesystem that doesn't support WAL's shared-memory locking (e.g. some network shares/NFS mounts) - this falls back to SQLite's default rollback journal -- **TMPDIR** is useful for mounting larger storage for temporary backup files (e.g., NFS) +- **TMPDIR** points the staging area for running backups somewhere else. Every backup is built there before it is uploaded, and a file backup needs roughly twice the size of its source, so either mount `/tmp` or set this to a mounted path with enough free space. Leaving it unmounted in Docker fills the container's writable layer - **TZ** only affects server-side logs. User-facing dates use the timezone from user profile settings - **PUID/PGID** control which UID/GID the application process runs as. Set these to match your host user (e.g., `PUID=1000 PGID=1000`) to avoid volume permission issues. The entrypoint adjusts the internal user at startup - **LOG_LEVEL** controls logging verbosity: diff --git a/docs/developer-guide/reference/versions.md b/docs/developer-guide/reference/versions.md index 0ad5ff29..ce2653d4 100644 --- a/docs/developer-guide/reference/versions.md +++ b/docs/developer-guide/reference/versions.md @@ -12,6 +12,7 @@ This document lists the database engines and versions supported by DBackup. | **MongoDB** | 4.x, 5.x, 6.x, 7.x, 8.x | `mongodump` | Standard operations | | **SQLite** | 3.x | `sqlite3` | File-based | | **Microsoft SQL Server** | 2017, 2019, 2022 | `mssql` npm | T-SQL commands | +| **Azure SQL Database** | Single database, elastic pool | `sqlpackage` | Beta, BACPAC export | | **Firebird** | 3.x, 4.x, 5.x | `gbak` | Beta, alias-based database list | ## Docker Container Tools @@ -175,6 +176,39 @@ WITH COMPRESSION, INIT; - `sa` credentials or appropriate backup permissions - Network access to SQL Server port (1433) +## Azure SQL Database + +### Supported Versions + +Azure SQL Database is versionless. It reports `12.0.2000` regardless of the engine actually running, so the adapter ignores the version entirely rather than deriving behaviour from it. + +Identified by `SERVERPROPERTY('EngineEdition')`, which is the only reliable signal: + +| EngineEdition | Product | Handled by | +| :--- | :--- | :--- | +| 5 | Azure SQL Database | This adapter | +| 8 | Azure SQL Managed Instance | Unsupported, rejected with a message | +| 6, 11 | Azure Synapse Analytics | Unsupported, rejected with a message | +| 9 | Azure SQL Edge | The MSSQL adapter | +| 1-4 | SQL Server | The MSSQL adapter | + +::: info Beta +The Azure SQL Database adapter is marked as Beta in the source type picker. +::: + +### Implementation + +Uses `sqlpackage` for a BACPAC export and import. Azure SQL Database has no `BACKUP DATABASE` statement, no server-scoped catalog views such as `sys.master_files`, and rejects three-part names, so every per-database catalog read opens its own connection. + +The export mechanism sits behind a `BacpacExporter` interface in `exporter/types.ts`. The Azure Import/Export REST API would fit the same seam and needs no binary, which mattered while it was unclear whether SqlPackage runs on arm64. It does, so only the SqlPackage implementation exists. + +```bash +sqlpackage /Action:Export /TargetFile:db.bacpac /SourceConnectionString:"..." +sqlpackage /Action:Import /SourceFile:db.bacpac /TargetConnectionString:"..." +``` + +There is no SSH mode. The service is a public endpoint, and SqlPackage runs in the DBackup container rather than on any host in between, so a tunnel would solve nothing. + ## Firebird ### Supported Versions diff --git a/docs/developer-guide/setup.md b/docs/developer-guide/setup.md index baa8b401..3322b976 100644 --- a/docs/developer-guide/setup.md +++ b/docs/developer-guide/setup.md @@ -17,37 +17,46 @@ Complete guide to setting up DBackup for development. - `mysql` / `mysqldump` (MySQL/MariaDB) - `psql` / `pg_dump` (PostgreSQL) - `mongodump` / `mongorestore` (MongoDB) - - `gbak` / `isql` (Firebird) - see [`scripts/setup-dev-macos.sh`](https://github.com/Skyfay/DBackup/blob/main/scripts/setup-dev-macos.sh) for a scripted macOS install (no Homebrew formula exists) + - `gbak` / `isql` (Firebird) + - `sqlpackage` (Azure SQL Database) + +Use the setup script for your platform rather than installing these by hand. Several of them have non-obvious requirements that a plain `brew install` gets wrong. ### macOS Installation ```bash -# Install Node.js via Homebrew brew install node - -# Install pnpm npm install -g pnpm -# Install database CLI tools -brew install mysql-client libpq mongodb-database-tools +# Installs every CLI tool the adapters need, and prints the PATH lines to add +./scripts/setup-dev-macos.sh +``` + +Then add this to `~/.zshrc`. The script prints it too, but it is easy to skip past: -# Add to PATH (add to ~/.zshrc) -export PATH="/opt/homebrew/opt/mysql-client/bin:$PATH" -export PATH="/opt/homebrew/opt/libpq/bin:$PATH" +```bash +export PATH="/opt/homebrew/opt/mysql-client/bin:/opt/homebrew/opt/postgresql@18/bin:/opt/homebrew/opt/postgresql@16/bin:/opt/homebrew/opt/postgresql@14/bin:/opt/homebrew/firebird-client/bin:$PATH" ``` +SqlPackage needs no entry of its own. The script installs it into `$(brew --prefix)/bin`, which is already on `PATH`, and wraps it so it finds the .NET runtime without a `DOTNET_ROOT` variable. + +::: warning Do not install `libpq` for PostgreSQL +`libpq` ships a `pg_dump` compiled without LZ4 and ZSTD support. If its directory comes first on `PATH`, PostgreSQL backups with native compression fail. Install the full `postgresql@XX` packages instead, which the script does. +::: + +::: tip Restart the dev server after changing PATH +A running `pnpm dev` inherited its environment at launch, and an editor-launched terminal often carries a different one than your login shell. If an adapter reports a missing CLI tool that works in your terminal, this is almost always why. +::: + ### Ubuntu/Debian Installation ```bash -# Install Node.js curl -fsSL https://deb.nodesource.com/setup_20.x | sudo -E bash - sudo apt-get install -y nodejs - -# Install pnpm npm install -g pnpm -# Install database CLI tools -sudo apt-get install mysql-client postgresql-client mongodb-database-tools +# Installs every CLI tool, including SqlPackage, and prints a summary of what resolved +sudo ./scripts/setup-dev-debian.sh ``` ## Clone and Install diff --git a/docs/index.md b/docs/index.md index c0c6cfd2..aac50b9d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -19,7 +19,7 @@ hero: features: - icon: ๐Ÿ—„๏ธ title: Multi-Database Support - details: Supports MySQL, MariaDB, PostgreSQL, MongoDB, SQLite, Redis, Valkey, Microsoft SQL Server, and Firebird (beta). + details: Supports MySQL, MariaDB, PostgreSQL, MongoDB, SQLite, Redis, Valkey, Microsoft SQL Server, Azure SQL Database (beta), and Firebird (beta). - icon: ๐Ÿ“ title: File & Folder Backups details: Back up an application's config and data directories alongside its database, in one job with one schedule and one retention policy. Folder tree picker, reusable exclude presets, Docker volumes read through the daemon, and VSS shadow copies on SMB sources. @@ -135,6 +135,7 @@ Then open [https://localhost:3000](https://localhost:3000) and create your first | **Valkey** | 7.2+ | Direct, SSH | Guided | | **SQLite** | 3.x | Local, SSH | Yes | | **Microsoft SQL Server** | 2017, 2019, 2022, Azure SQL Edge | Direct, SSH | Yes | +| **Azure SQL Database** (Beta) | Single database, elastic pool | Direct | Yes (drops the target first) | | **Firebird** (Beta) | 3.x, 4.x, 5.x | Direct, SSH | Yes (pre-configured aliases) | == ๐Ÿ“ Directory Sources diff --git a/docs/user-guide/destinations/index.md b/docs/user-guide/destinations/index.md index 86c715d0..760a038b 100644 --- a/docs/user-guide/destinations/index.md +++ b/docs/user-guide/destinations/index.md @@ -76,6 +76,46 @@ Backups are organized by job name with sidecar metadata files: The `.meta.json` file stores compression, encryption metadata (IV, auth tag, profile ID), database version, and timestamp. +## Upload Performance (S3) + +A backup is one large file, so the only way to use more of a fast link is to send several pieces of it at the same time. Every S3 destination (Amazon S3, Cloudflare R2, Hetzner Object Storage, S3-Compatible) splits an upload into parts and sends **8 parts of 8 MB** at once by default. + +Both values are adjustable per destination under **Configuration โ†’ Parallel Upload Parts**: + +| Field | Description | Default | Range | +| :--- | :--- | :--- | :--- | +| **Parts at once** | Parts uploaded simultaneously | `8` | 1 to 32 | +| **Max part size (MB)** | Upper bound on the size of each part | `8` | 5 to 64 | + +### Parts at once is the speed setting + +Each part travels over its own connection, and a single connection to an object store is usually capped somewhere between 5 and 10 MB/s no matter how much bandwidth is available. Total throughput is therefore roughly `parts at once x per-connection speed`, so raising this is what makes an upload faster. + +Raise it when the upload speed in the run log is well below what the server's link can do. Lower it if the provider starts answering with `SlowDown` or `503`. + +::: tip Measured example +Uploading a 1.39 GB backup to Cloudflare R2 over a 10 Gbit link, where each connection managed about 5.9 MB/s: + +| Parts at once | Throughput | +| :--- | :--- | +| 4 | 26 MB/s | +| 32 | 187 MB/s | +::: + +### Max part size is a memory setting + +Parts in flight are held in memory, so an upload uses roughly `(parts + 1) x part size` while it runs. The form shows the figure as you change the values. The default works out to about 72 MB, the maximum to about 2 GB. + +You are setting an upper bound, not a fixed size. **DBackup picks the largest part size at or below your value that still gives every connection something to upload.** The right size depends on how large a backup turns out to be, and that differs from run to run, so it is not something a static setting can track. + +Why it matters: with 32 parts at once, a 1.39 GB backup split into 64 MB parts is only 21 parts. Eleven connections would get nothing, and the upload would take as long as its single slowest part. The automatic adjustment prevents that, and the run log reports the size actually used. + +::: warning Concurrent jobs +If **Max Concurrent Jobs** is above 1, several uploads can run at once and each one uses its own memory budget. Destinations within a single job upload one after another, so those do not add up. +::: + +DBackup also raises the part size above your maximum in one case: when an archive is so large that your value would need more than S3's limit of 10,000 parts. The alternative there is an upload the service rejects. + ## Retention Policies Destinations work with retention policies to automatically clean up old backups: diff --git a/docs/user-guide/destinations/s3-aws.md b/docs/user-guide/destinations/s3-aws.md index 773b4a22..81c75de2 100644 --- a/docs/user-guide/destinations/s3-aws.md +++ b/docs/user-guide/destinations/s3-aws.md @@ -15,6 +15,8 @@ Amazon S3 requires a [Credential Profile](/user-guide/security/credential-profil | **Bucket** | S3 bucket name | - | โœ… | | **Primary Credential** | `ACCESS_KEY` credential profile (Access Key ID + Secret Access Key) | - | โœ… | | **Path Prefix** | Folder path within the bucket | - | โŒ | +| **Parts at once** | Upload parts sent simultaneously ([details](/user-guide/destinations/#upload-performance-s3)) | `8` | โŒ | +| **Max part size (MB)** | Upper bound on the size of each upload part | `8` | โŒ | | **Storage Class** | S3 storage class for uploaded objects | `STANDARD` | โŒ | ### Storage Classes @@ -69,7 +71,7 @@ Instead of `AmazonS3FullAccess`, scope permissions to a single bucket: ## How It Works -- Backups upload via the AWS SDK using multipart upload for large files +- Backups upload via the AWS SDK as 8 parallel parts by default ([details](/user-guide/destinations/#upload-performance-s3)) - All credentials are stored AES-256-GCM encrypted in the database - Storage class is set per-object at upload time - The Path Prefix creates a virtual folder structure within your bucket @@ -102,7 +104,7 @@ The AWS Access Key Id you provided does not exist in our records ### Slow Uploads / Timeout -**Solution:** Choose a region geographically close to your DBackup server. For large backups, ensure your server has sufficient upload bandwidth. +**Solution:** Check the upload speed the run log reports at the end of the upload. If it is well below what the server's link can do, raise **Parts at once** under Configuration ([details](/user-guide/destinations/#upload-performance-s3)). A single connection to S3 often tops out around 5 to 10 MB/s no matter how much bandwidth is available. Also choose a region geographically close to your DBackup server. ## Next Steps diff --git a/docs/user-guide/destinations/s3-generic.md b/docs/user-guide/destinations/s3-generic.md index f70f51cd..0fa24f58 100644 --- a/docs/user-guide/destinations/s3-generic.md +++ b/docs/user-guide/destinations/s3-generic.md @@ -17,6 +17,8 @@ S3-Compatible Storage requires a [Credential Profile](/user-guide/security/crede | **Primary Credential** | `ACCESS_KEY` credential profile (Access Key ID + Secret Access Key) | - | โœ… | | **Force Path Style** | Use path-style URLs (`endpoint/bucket`) instead of virtual-hosted | `false` | โŒ | | **Path Prefix** | Folder path within the bucket | - | โŒ | +| **Parts at once** | Upload parts sent simultaneously ([details](/user-guide/destinations/#upload-performance-s3)) | `8` | โŒ | +| **Max part size (MB)** | Upper bound on the size of each upload part | `8` | โŒ | ::: tip Force Path Style Enable this for providers that don't support virtual-hosted-style URLs (e.g. MinIO, Ceph). When enabled, requests go to `endpoint/bucket/key` instead of `bucket.endpoint/key`. @@ -90,7 +92,7 @@ services: ## How It Works - Uses the S3-compatible API via the AWS SDK -- Multipart upload for large files +- Backups upload as 8 parallel parts by default ([details](/user-guide/destinations/#upload-performance-s3)) - All credentials are stored AES-256-GCM encrypted in the database ## Troubleshooting diff --git a/docs/user-guide/destinations/s3-hetzner.md b/docs/user-guide/destinations/s3-hetzner.md index 43b45b08..544b3c86 100644 --- a/docs/user-guide/destinations/s3-hetzner.md +++ b/docs/user-guide/destinations/s3-hetzner.md @@ -15,6 +15,8 @@ Hetzner Object Storage requires a [Credential Profile](/user-guide/security/cred | **Bucket** | Bucket name | - | โœ… | | **Primary Credential** | `ACCESS_KEY` credential profile (Access Key + Secret Key) | - | โœ… | | **Path Prefix** | Folder path within the bucket | - | โœ… | +| **Parts at once** | Upload parts sent simultaneously ([details](/user-guide/destinations/#upload-performance-s3)) | `8` | โŒ | +| **Max part size (MB)** | Upper bound on the size of each upload part | `8` | โŒ | ### Regions @@ -43,7 +45,7 @@ Unlike other S3 adapters, Hetzner Object Storage **requires** a Path Prefix. Set ## How It Works - DBackup connects to `https://..your-objectstorage.com` automatically -- Uses S3-compatible API - uploads via multipart for large files +- Uses S3-compatible API, uploading a backup as 8 parallel parts by default ([details](/user-guide/destinations/#upload-performance-s3)) - All credentials are stored AES-256-GCM encrypted in the database ## Troubleshooting diff --git a/docs/user-guide/destinations/s3-r2.md b/docs/user-guide/destinations/s3-r2.md index ebde6a01..360c4921 100644 --- a/docs/user-guide/destinations/s3-r2.md +++ b/docs/user-guide/destinations/s3-r2.md @@ -16,6 +16,8 @@ Cloudflare R2 requires a [Credential Profile](/user-guide/security/credential-pr | **Jurisdiction** | Bucket jurisdiction: `Standard`, `EU`, or `FedRAMP` | `Standard` | โŒ | | **Primary Credential** | `ACCESS_KEY` credential profile (R2 API token Access Key ID + Secret) | - | โœ… | | **Path Prefix** | Folder path within the bucket | - | โŒ | +| **Parts at once** | Upload parts sent simultaneously ([details](/user-guide/destinations/#upload-performance-s3)) | `8` | โŒ | +| **Max part size (MB)** | Upper bound on the size of each upload part | `8` | โŒ | ::: warning EU Jurisdiction Buckets created with the EU jurisdiction use the `*.eu.r2.cloudflarestorage.com` endpoint. If the jurisdiction setting does not match your bucket's actual location, you will get "Access Denied" or "bucket does not exist" errors. @@ -42,7 +44,7 @@ R2 has no egress fees, making it ideal for backups you may need to restore frequ ## How It Works - DBackup connects to the R2 endpoint `https://.r2.cloudflarestorage.com` automatically -- Uses S3-compatible API - uploads via multipart for large files +- Uses S3-compatible API, uploading a backup as 8 parallel parts by default ([details](/user-guide/destinations/#upload-performance-s3)) - All credentials are stored AES-256-GCM encrypted in the database ## Troubleshooting diff --git a/docs/user-guide/features/file-backups.md b/docs/user-guide/features/file-backups.md index c857d6c3..245819f2 100644 --- a/docs/user-guide/features/file-backups.md +++ b/docs/user-guide/features/file-backups.md @@ -25,6 +25,10 @@ the DBackup host. Two consequences follow, and both scale with the size of the s - **Every byte crosses the network twice** - source โ†’ DBackup โ†’ destination - because nothing runs on the source machine. +::: warning Running in Docker +That temporary space is the container's writable layer unless it is mounted. A job large enough can fill the Docker disk before the archive is finished. Mount `/tmp` to a volume with room to spare, or set `TMPDIR` to another mounted path, before backing up a large directory. See [Volume Mounts](/user-guide/installation#volume-mounts). +::: + Incremental mode softens the first run's cost on later runs (unchanged files are not fetched at all and are carried into the new archive by reference), but the first full run pays it in full, and so does every scheduled full backup after it. diff --git a/docs/user-guide/features/restore.md b/docs/user-guide/features/restore.md index d9d11d1d..f5923684 100644 --- a/docs/user-guide/features/restore.md +++ b/docs/user-guide/features/restore.md @@ -191,6 +191,17 @@ RESTORE DATABASE [dbname] FROM DISK = '/path/backup.bak' - Requires shared volume - Uses T-SQL commands +### Azure SQL Database + +```bash +sqlpackage /Action:Import /SourceFile:backup.bacpac /TargetConnectionString:"..." +``` + +- A BACPAC import always creates the database, so restoring onto an existing name **drops it first** +- Azure keeps a dropped database recoverable through **Deleted databases** on the logical server +- Most of the runtime is Azure creating the database, not moving data, so a small database takes about as long as a large one +- See [Azure SQL Database source](/user-guide/sources/azure-sql) for the full caveats + ### Firebird ```bash diff --git a/docs/user-guide/getting-started.md b/docs/user-guide/getting-started.md index e2334c01..11d990a3 100644 --- a/docs/user-guide/getting-started.md +++ b/docs/user-guide/getting-started.md @@ -8,7 +8,7 @@ DBackup is a self-hosted web application for automating database backups. It sup ## Key Features -- **Multi-Database Support**: MySQL, MariaDB, PostgreSQL, MongoDB, SQLite, Redis, Valkey, Microsoft SQL Server, Firebird (beta) +- **Multi-Database Support**: MySQL, MariaDB, PostgreSQL, MongoDB, SQLite, Redis, Valkey, Microsoft SQL Server, Azure SQL Database (beta), Firebird (beta) - **Flexible Storage**: 13+ adapters including local filesystem, S3, Google Drive, SFTP, and more - **Multi-Destination Jobs**: A single job can upload to multiple storage destinations simultaneously - **Backup Encryption**: AES-256-GCM encryption with an Encryption Vault, key rotation, and offline Recovery Kits diff --git a/docs/user-guide/installation.md b/docs/user-guide/installation.md index b99b795d..9c58c29b 100644 --- a/docs/user-guide/installation.md +++ b/docs/user-guide/installation.md @@ -34,6 +34,7 @@ services: volumes: - ./data:/data # All persistent data (db, storage, certs) - ./backups:/backups # Optional: used for local backups + # - ./tmp:/tmp # Recommended: staging space for running backups, keeps them off the Docker disk ``` ```bash [Docker Run] @@ -46,6 +47,7 @@ docker run -d \ -e BETTER_AUTH_URL="https://localhost:3000" \ -v "$(pwd)/data:/data" \ -v "$(pwd)/backups:/backups" \ + -v "$(pwd)/tmp:/tmp" \ skyfay/dbackup:latest ``` @@ -90,7 +92,7 @@ Access the application at [https://localhost:3000](https://localhost:3000) (acce | `DATABASE_URL` | โŒ | SQLite path. Default: `file:/data/db/dbackup.db` | | `SQLITE_WAL_MODE` | โŒ | Set to `false` to disable WAL mode. Default: `true` | | `TZ` | โŒ | Server timezone for logs. Default: `UTC` | -| `TMPDIR` | โŒ | Temp directory for large backups. Default: `/tmp` | +| `TMPDIR` | โŒ | Temp directory for large backups, see [Volume Mounts](#volume-mounts). Default: `/tmp` | | `LOG_LEVEL` | โŒ | Logging verbosity: `debug`, `info`, `warn`, `error`. Default: `info` | | `DISABLE_HTTPS` | โŒ | Set to `true` to use plain HTTP. Default: `false` (HTTPS). **Set this when running behind a reverse proxy** - see [Reverse Proxy Setup](#reverse-proxy-setup) | | `PUID` | โŒ | User ID the container runs as. Default: `1001` | @@ -198,6 +200,13 @@ secrets: | :--- | :---: | :--- | | `/data` | โœ… | All persistent data (database, uploads, certificates) | | `/backups` | โŒ | Optional: used for local backups | +| `/tmp` | โŒ | Recommended: staging space while a backup is being built | + +::: warning Give `/tmp` its own mount +Every backup is written to a temporary directory first and only uploaded to the destination afterwards. For file backups that needs roughly twice the size of the source, since the tree is staged to disk and the archive is written next to it before either is cleaned up. + +Without a mount, all of that lands inside the container's writable layer on the Docker disk, which is usually far smaller than the data being backed up. Mount `/tmp` to a location with enough free space, or point `TMPDIR` at another mounted path. +::: ::: info SQLite WAL mode DBackup runs its internal SQLite database in [WAL (Write-Ahead Logging)](https://www.sqlite.org/wal.html) mode by default for better read/write concurrency. This creates two extra files next to the database, `dbackup.db-wal` and `dbackup.db-shm`, inside `/data/db`. They are normal and required while the app is running - do not delete them manually. Back up or copy the whole `db` folder together (not just the `.db` file) to avoid losing uncommitted writes. Set `SQLITE_WAL_MODE=false` to disable WAL mode if your storage backend doesn't support it (e.g. some network shares/NFS mounts). diff --git a/docs/user-guide/jobs/retention.md b/docs/user-guide/jobs/retention.md index 87b12450..5b2648a0 100644 --- a/docs/user-guide/jobs/retention.md +++ b/docs/user-guide/jobs/retention.md @@ -4,7 +4,7 @@ Automatically manage backup storage by defining how long to keep backups. ## Overview -Retention policies prevent unlimited storage growth by automatically deleting old backups. DBackup supports two retention modes: +Retention policies prevent unlimited storage growth by automatically deleting old backups. DBackup supports three retention modes: | Mode | Description | Best For | | :--- | :--- | :--- | @@ -58,6 +58,7 @@ With `Keep Count: 5`: ## Smart Retention (GFS) Grandfather-Father-Son is an intelligent retention strategy that keeps: +- The most recent backups (hourly, optional) - Recent backups (daily) - Some older backups (weekly) - Fewer old backups (monthly) @@ -67,42 +68,66 @@ Grandfather-Father-Son is an intelligent retention strategy that keeps: | Setting | Description | Example | | :--- | :--- | :--- | +| **Hourly** | Hourly backups to keep, off unless enabled | `24` | | **Daily** | Days to keep daily backups | `7` | | **Weekly** | Weeks to keep weekly backups | `4` | | **Monthly** | Months to keep monthly backups | `12` | | **Yearly** | Years to keep yearly backups | `3` | +### Hourly Tier + +Most schedules do not need an hourly tier, so the field is hidden until you ask for it. In the policy form, click **Add hourly tier** below the tier inputs. It starts at `24` and can be removed again with **Remove hourly tier**, which sets it back to `0`. + +A policy that already has an hourly value opens with the field visible. Policies created before this tier existed keep exactly the same deletion behaviour, because a missing value counts as `0`. + +::: tip Sub-hourly schedules +A job running every 15 minutes with **Hourly: 24** keeps one backup per hour and deletes the other three. That is the point of the tier, but it is a change in outcome if you are switching over from **Simple** retention. +::: + ### How It Works -The algorithm evaluates each backup: +Each tier keeps the newest backup of every time bucket it covers, working from the newest backup backwards. Buckets that a finer tier already covers are skipped, and a tier only counts what it adds itself. + +1. **Hourly bucket**: the newest backup of each of the last N hours that have backups +2. **Daily bucket**: the newest backup of each of the next N days +3. **Weekly bucket**: the newest backup of each of the next N weeks +4. **Monthly bucket**: the newest backup of each of the next N months +5. **Yearly bucket**: the newest backup of each of the next N years + +::: warning The tiers add up, they do not overlap +**Daily: 7** means seven days *on top of* what the hourly tier already covers, not seven days in total. With **Hourly: 24, Daily: 7** the policy reaches back roughly nine days and keeps about 31 backups. -1. **Daily bucket**: Is this one of the last N days' backups? -2. **Weekly bucket**: Is this the most recent backup from the last N weeks? -3. **Monthly bucket**: Is this the most recent backup from the last N months? -4. **Yearly bucket**: Is this the most recent backup from the last N years? +restic and borg read the same numbers as a union, where the daily window includes the hours the hourly tier covers. A config copied from one of those tools keeps more in DBackup than it does there. +::: + +A tier counts buckets that **have** backups, not wall-clock time. If a run is skipped, **Hourly: 24** reaches further back than 24 hours rather than losing a slot. -A backup is kept if it qualifies for **any** bucket. +::: warning Daylight saving time +Buckets are built in the timezone configured under Settings. When the clock goes back, the repeated hour maps to a single hourly bucket, so one backup loses its slot once a year. Day, week, month and year buckets are unaffected. +::: ### Example Timeline Configuration: Daily=7, Weekly=4, Monthly=12, Yearly=2 After 1 year of daily backups: -- **Days 1-7**: All 7 daily backups kept -- **Weeks 2-4**: 1 backup per week (3 more) -- **Months 2-12**: 1 backup per month (11 more) -- **Previous year**: 1 backup kept +- **Daily**: the 7 newest days +- **Weekly**: 4 further weeks, starting after the weeks the daily tier already covers +- **Monthly**: 12 further months, starting after the months covered so far +- **Yearly**: 2 further years + +**Total**: 25 backups instead of 365, the sum of the four tiers. -**Total**: ~22 backups instead of 365! +Adding **Hourly: 24** to the same policy keeps roughly 24 more, and the daily tier then starts after the hours it covers rather than at today. ### Visual Example ``` -Today 7 days ago 1 month ago 1 year ago - | | | | - โ–ผ โ–ผ โ–ผ โ–ผ -[โ– ][โ– ][โ– ][โ– ][โ– ][โ– ][โ– ] [โ– ] [โ– ] [โ– ]... [โ– ] - โ””โ”€โ”€ Daily โ”€โ”€โ”˜ โ”” Weekly โ”˜ โ””โ”€โ”€ Monthly โ”€โ”€โ”˜ +Now last 24h 7 days weeks months years + | | | | | | + โ–ผ โ–ผ โ–ผ โ–ผ โ–ผ โ–ผ +[โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– โ– ][โ– ][โ– ][โ– ][โ– ][โ– ][โ– ][โ– ] [โ– ] [โ– ] [โ– ]... [โ– ] [โ– ] + โ””โ”€โ”€ Hourly โ”€โ”€โ”˜โ””โ”€โ”€โ”€โ”€โ”€ Daily โ”€โ”€โ”€โ”€โ”€โ”˜ โ”” Weekly โ”˜ โ”” Monthly โ”˜ โ”” Yearly โ”˜ ``` ## Locked Backups @@ -136,7 +161,7 @@ Monthly: 24 Yearly: 5 ``` -Keeps ~50 backups over 5 years. +Keeps 51 backups over 5 years. ### Moderate (Balanced) @@ -147,7 +172,7 @@ Monthly: 12 Yearly: 2 ``` -Keeps ~25 backups over 2 years. +Keeps 25 backups over 2 years. ### Aggressive (Minimal) @@ -158,7 +183,18 @@ Monthly: 6 Yearly: 1 ``` -Keeps ~12 backups over 1 year. +Keeps 12 backups over 1 year. + +### Hourly Schedule + +``` +Hourly: 24 +Daily: 7 +Weekly: 4 +Monthly: 12 +``` + +Keeps 47 backups and gives a full day at hourly resolution before the daily tier takes over. ## Retention Execution @@ -166,7 +202,7 @@ Retention runs as the **final step** of each backup job, applied **per destinati 1. Backup upload completes for a destination 2. List all backups for this job in that specific destination -3. Read metadata (check lock status) +3. Read each backup's metadata sidecar for its lock status, chain and creation time 4. Apply that destination's retention policy 5. Delete expired backups 6. Repeat for each remaining destination @@ -175,6 +211,14 @@ Retention runs as the **final step** of each backup job, applied **per destinati Retention is skipped for any destination where the upload failed. This prevents deleting old backups when the new backup didn't arrive. ::: +### Which Time a Backup Is Judged By + +Backups are sorted into buckets by the creation time **DBackup recorded when it wrote the backup**, which is stored in the backup's `.meta.json` sidecar. The file's modification time on the destination is only used when there is no sidecar, for backups taken before this was recorded or for destinations DBackup cannot read files from. + +This matters because a modification time is easy to lose. Copying the backup directory without preserving timestamps, moving it to another server, or restoring it from a backup of its own stamps every file with the current time. Judged by that, the entire history collapses into a single bucket and the next retention pass deletes all but one backup from it. + +The run log names any backup whose two times disagree by more than an hour, so a destination in that state is visible before it costs anything. + ## Compliance Considerations ### GDPR @@ -201,7 +245,7 @@ Retention is skipped for any destination where the upload failed. This prevents | Schedule | Recommended Retention | | :--- | :--- | -| Hourly | Daily: 24-48 | +| Hourly | Hourly: 24-48, plus Daily: 7-14 | | Daily | Daily: 7-14 | | Weekly | Weekly: 4-8 | | Monthly | Monthly: 12-24 | @@ -250,6 +294,10 @@ With compression (70% reduction): 3. View job logs for retention step 4. Ensure backup ran successfully +### Everything Was Deleted After Moving a Destination + +Check the retention step in the run log for warnings about backups whose recorded creation time disagrees with the destination's modification time. Backups written before DBackup recorded a creation time fall back to the modification time, and a move that did not preserve timestamps puts all of them in the same bucket. Lock the backups you cannot lose before moving a destination. + ### Too Many Backups Deleted 1. Check retention settings @@ -259,9 +307,9 @@ With compression (70% reduction): ### Wrong Backups Deleted -The GFS algorithm keeps the **oldest** backup in each time bucket. This is intentional: -- Weekly: Keeps backup from start of week -- Monthly: Keeps backup from start of month +The GFS algorithm keeps the **newest** backup in each time bucket. A week with seven backups keeps the one from the end of the week, not the start. + +If a destination holds more backups than the policy allows, check the retention step in the run log. Locked backups and incremental chains are both kept beyond the policy, and the log names them. ## API Reference @@ -275,6 +323,7 @@ Retention configuration in job: "keepCount": 5 }, "smart": { + "hourly": 24, "daily": 7, "weekly": 4, "monthly": 12, @@ -284,6 +333,8 @@ Retention configuration in job: } ``` +`hourly` may be omitted. A missing value counts as `0` and disables the tier. + ## Next Steps - [Creating Jobs](/user-guide/jobs/) - Configure backup jobs diff --git a/docs/user-guide/security/compression.md b/docs/user-guide/security/compression.md index 931b9d3b..b64d63e8 100644 --- a/docs/user-guide/security/compression.md +++ b/docs/user-guide/security/compression.md @@ -105,7 +105,7 @@ counts as `gz`. | Audio | `aac` `ape` `flac` `m4a` `mka` `mp3` `oga` `ogg` `opus` `wma` | | Images | `avif` `gif` `heic` `heif` `jp2` `jpeg` `jpg` `jxl` `png` `webp` | | Archives | `7z` `br` `bz2` `cab` `gz` `lz4` `lzma` `rar` `tbz2` `tgz` `txz` `xz` `zip` `zst` | -| ZIP containers | `apk` `docx` `epub` `ipa` `jar` `nupkg` `odp` `ods` `odt` `pptx` `vsix` `war` `whl` `xlsx` `xpi` | +| ZIP containers | `apk` `bacpac` `docx` `epub` `ipa` `jar` `nupkg` `odp` `ods` `odt` `pptx` `vsix` `war` `whl` `xlsx` `xpi` | | Web fonts | `woff` `woff2` | | Encrypted | `age` `enc` `gpg` `pgp` | | Disk images | `dmg` | diff --git a/docs/user-guide/sources/azure-sql.md b/docs/user-guide/sources/azure-sql.md new file mode 100644 index 00000000..4094ac7e --- /dev/null +++ b/docs/user-guide/sources/azure-sql.md @@ -0,0 +1,109 @@ +# Azure SQL Database + +Azure SQL Database is Microsoft's managed PaaS database. DBackup backs it up by exporting a BACPAC with SqlPackage, which ships inside the DBackup container image. + +::: warning This is not a native backup +Azure SQL Database has no `BACKUP DATABASE` statement, so a native `.bak` cannot be produced from it. A BACPAC is a schema and data export, and Microsoft states it is not intended as a backup and restore mechanism. Azure's own automated point-in-time restore remains your primary recovery path. Use this adapter for what Azure does not give you: a copy that lives outside Azure, in a format you control. +::: + +## Which product this covers + +| Engine | Supported | Use instead | +| :--- | :--- | :--- | +| Azure SQL Database (single database, elastic pool) | โœ… | - | +| Azure SQL Managed Instance | โŒ | Not supported by DBackup | +| SQL Server 2017/2019/2022, Azure SQL Edge | โŒ | [Microsoft SQL Server](/user-guide/sources/mssql) | +| Azure Synapse Analytics | โŒ | Not supported by DBackup | + +Both source types accept the same connection, so a wrong choice is easy to make. Each one detects the engine and refuses with a message naming what it actually found. + +## Prerequisites + +- A SQL login on the logical server. Microsoft Entra authentication is not supported by this adapter. +- A firewall rule allowing the IP address DBackup connects from. Under **Networking** on the logical server in the Azure portal. +- To restore, a login that can create databases: a member of `dbmanager` in `master`, or the server administrator. + +No tooling to install. SqlPackage is part of the DBackup image on both `linux/amd64` and `linux/arm64`. + +## Configuration + +::: info Credential Profiles +A `USERNAME_PASSWORD` credential profile is required. This adapter has no SSH mode: Azure SQL Database is a public endpoint, and the export runs inside the DBackup container rather than on any host in between. +::: + +| Field | Description | Default | Required | +| :--- | :--- | :--- | :--- | +| **Host** | Logical server name, e.g. `myserver.database.windows.net` | - | โœ… | +| **Port** | Server port | `1433` | โœ… | +| **Primary Credential** | `USERNAME_PASSWORD` credential profile | - | โœ… | +| **Request Timeout** | Timeout in ms for catalog queries. The export itself is never timed out. | `300000` | โŒ | + +Encryption is always on and the server certificate is always verified. Neither is configurable, because Azure presents a valid certificate on every connection. + +## Backup file format + +A single database is stored as a `.bacpac`, which is a ZIP containing `model.xml` and the table data. Because it is already compressed, DBackup stores it as-is and skips its own compression pass. + +Selecting several databases produces one BACPAC per database, packed into a `.tar` with a manifest. The manifest is what lets the restore screen list the databases without downloading the archive. + +## Consistency + +A BACPAC export runs as ordinary queries against a live database. It is **not** a point-in-time snapshot, and Microsoft documents that an export taken while the database is being written to can be internally inconsistent, in a way that only surfaces when the import later fails. + +DBackup writes this caveat into the run log of every backup. To get a guaranteed-consistent export, either pause writes for the duration, or export from a copy you make yourself: + +```sql +CREATE DATABASE myapp_snapshot AS COPY OF myapp; +``` + +Point the DBackup source at the copy, and drop it when the backup finishes. + +::: warning Ledger tables +If a database uses [Ledger](https://learn.microsoft.com/azure/azure-sql/database/ledger-overview), a BACPAC cannot capture its history tables or its generated-always columns. The tamper evidence Ledger exists to provide is therefore **not** part of the backup. DBackup raises this as a warning in the run log when it detects one. +::: + +## Restore + +::: danger Restoring over a database drops it first +A BACPAC import always creates the database and has no overwrite mode. Restoring onto an existing name therefore **drops that database** before importing, which is how every other source in DBackup behaves on a restore. Everything in it is gone, including any data written since the backup. + +Azure keeps a dropped database recoverable through **Deleted databases** on the logical server, for the retention window of its own automated backups. That is a safety net, not a plan. + +Pick **Restore to a new database** in the restore dialog if you want the existing one left alone. +::: + +A backup taken from Azure SQL Database can only be restored to an Azure SQL Database source. Restoring it into a Microsoft SQL Server source is blocked, even though a BACPAC would technically import into on-premises SQL Server. + +The new database is created at the service tier the import defaults to. Check and adjust it in the Azure portal afterwards if the tier matters for your workload. + +::: info A restore takes minutes regardless of size +Most of the time goes into Azure creating the database, not into moving your data. A measured restore of a 3 KB BACPAC took just over two minutes, of which 113 seconds were spent on `Updating database` before a single row was written. That cost is roughly constant, so a large database takes about the same two minutes plus the time its data actually needs. +::: + +## Troubleshooting + +### Client with IP address is not allowed to access the server + +Azure's firewall rejected the connection before authentication. Add a rule for the address DBackup connects from, under **Networking** on the logical server. In a container this is the public address of the Docker host, not the container's own address. + +### Login failed for user + +The login exists on the server but not in the database being exported, or the password is wrong. A login that can connect to `master` still needs a user in each database it is meant to export. + +### The first export after a quiet period takes minutes + +The first connection to a serverless database resumes it from auto-pause, and the schema extraction runs many small catalog queries against a database that is still warming up. A cold export measured just under four minutes where the warm one took six seconds. On Basic and low-tier Standard databases the export stays throttled by the service tier itself. + +### Restore failed and the target database is gone + +The drop succeeded and the import did not. Recover the database through **Deleted databases** on the logical server in the Azure portal, then read the run log for why the import failed before trying again. + +### Export fails on a large table + +Microsoft documents export failures on large tables that have no clustered index with non-null values. Adding one, or exporting from a copy taken during a quiet period, is the usual fix. + +## See Also + +- [Restore Guide](/user-guide/features/restore) - General restore documentation +- [Encryption](/user-guide/security/encryption) - Encrypting your backups +- [Microsoft SQL Server](/user-guide/sources/mssql) - For SQL Server and Azure SQL Edge diff --git a/docs/user-guide/sources/index.md b/docs/user-guide/sources/index.md index 99d3cc4f..ad109d0c 100644 --- a/docs/user-guide/sources/index.md +++ b/docs/user-guide/sources/index.md @@ -14,6 +14,7 @@ DBackup supports a wide variety of database engines. | [Valkey](/user-guide/sources/valkey) | 7.2+ | `redis-cli --rdb` | Manual | | [SQLite](/user-guide/sources/sqlite) | 3.x | `.dump` command | โœ… | | [MSSQL](/user-guide/sources/mssql) | 2017, 2019, 2022 | `BACKUP DATABASE` | โœ… | +| [Azure SQL Database](/user-guide/sources/azure-sql) | Single database, elastic pool | `SqlPackage` BACPAC export | โœ… (drops the target first) | | [Firebird](/user-guide/sources/firebird) | 3.x, 4.x, 5.x | `gbak` | โœ… (pre-configured aliases) | ## Directory Sources diff --git a/docs/user-guide/sources/mssql.md b/docs/user-guide/sources/mssql.md index 3391d9d3..1f51d9fe 100644 --- a/docs/user-guide/sources/mssql.md +++ b/docs/user-guide/sources/mssql.md @@ -192,6 +192,96 @@ If SQL Server is installed **directly on the host** (bare-metal/VM), you can use | **Private Key** | PEM-format private key (optionally with passphrase) | | **Agent** | Uses the system SSH agent (`SSH_AUTH_SOCK`) | +## SQL Server on Windows + +**Backup Path (Server)** is handed to SQL Server exactly as written, so on a Windows server it has to be a Windows path. Both forms are accepted: + +| Form | Example | +| :--- | :--- | +| Local drive | `D:/SQLBackup` | +| UNC share | `\\192.168.0.10\SQLBackup` | + +Write a drive path with **forward slashes**. Windows accepts either separator in `BACKUP DATABASE`, and forward slashes are also what SFTP expects, so one spelling works in every transfer mode. + +::: danger Change the default path first +The default `/var/opt/mssql/backup` is a Linux path. Windows treats it as relative to the instance's own backup directory and the backup fails on the first run: + +``` +Cannot open backup device +'D:\Program Files\Microsoft SQL Server\MSSQL15.MSSQLSERVER\MSSQL\Backup\/var/opt/mssql/backup/mydb.bak' +Operating system error 3 (The system cannot find the path specified.) +``` + +In **local** file transfer mode nothing catches this before the run. The connection test checks the SQL Server connection, and the path is only used once a backup starts. In **SSH** mode the test does check the path, because it can reach it. +::: + +### SSH mode + +The `.bak` file is written on the Windows side and has to travel back, and SSH mode is what that is for. It behaves exactly as on Linux: DBackup tunnels the SQL Server connection through SSH and the file comes back over the same connection. Nothing is shared, port 1433 does not have to be reachable, and **Backup Path (Server)** stays an ordinary local directory on the server. + +On the SQL Server host DBackup only ever uses SFTP and port forwarding, never a remote command, so the default `cmd.exe` shell does not matter. + +Windows Server 2019 and newer ship the OpenSSH server as an optional feature. Install and start it in PowerShell as Administrator: + +```powershell +Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0 +Start-Service sshd +Set-Service -Name sshd -StartupType Automatic +``` + +Then set up the source as described under [Connection Modes](#connection-modes) and point **Backup Path (Server)** at the directory SQL Server writes into, for example `D:/SQLBackup`. **Test Connection** writes a probe file over SFTP and asks SQL Server whether it sees it, so a wrong path is reported before the first backup. + +::: warning Administrator accounts keep their public keys elsewhere +When the SSH account belongs to the local **Administrators** group, OpenSSH on Windows reads `C:\ProgramData\ssh\administrators_authorized_keys` and ignores that user's own `authorized_keys`. See [key management in the OpenSSH documentation](https://learn.microsoft.com/windows-server/administration/openssh/openssh_keymanagement). + +A normal account avoids the whole question and is enough. It needs read, write and delete access to the backup directory and no SQL Server privileges at all. +::: + +### Local mode over an SMB share + +Use this where the OpenSSH server is not an option, on Windows Server 2016 and older or where policy rules it out. It also fits when the `.bak` belongs on a NAS rather than on the server's own disk. + +``` +DBackup (Docker on Synology/Linux) Windows Server + โ”‚ โ””โ”€โ”€ SQL Server + โ”‚ BACKUP DATABASE (TCP 1433) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ–บ โ”‚ + โ”‚ โ”‚ writes + โ”‚ โ–ผ + โ””โ”€โ”€ reads /mnt/sql-backup โ—„โ”€โ”€โ”€โ”€ \\synology-nas\sql-backup + (the same share, seen from both sides) +``` + +1. Create the share on the NAS or file server, for example `sql-backup`. +2. Mount it into the DBackup container, for example at `/mnt/sql-backup`. +3. Set **File Transfer Mode** to `local`. +4. Set **Backup Path (Server)** to the UNC path the SQL Server uses, for example `\\synology-nas\sql-backup`. +5. Set **Local Backup Path** to the mount point inside the DBackup container, for example `/mnt/sql-backup`. + +The two paths point at the same directory from two sides, exactly as in the Docker volume setup above. DBackup never speaks SMB itself, it reads the mount. + +::: warning The SQL Server service account needs access to the share +`BACKUP DATABASE` runs as the **SQL Server service account**, not as the login DBackup connects with. The default service accounts (`NT Service\MSSQLSERVER`, `Network Service`, `Local System`) have no identity on the network, so writing to a UNC path fails with operating system error 5 even when the share is open to every account you tested it with. + +Two ways out: + +- Run the SQL Server service under a **domain account** and give that account write access to the share. +- Grant the share and NTFS permissions to the **computer account** (`DOMAIN\SERVERNAME$`), which is the identity a default service account presents on the network. + +A local Windows account on the SQL Server is not enough, the file server has no way to authenticate it. +::: + +::: tip Verify the path from the server before configuring the source +Run this in SSMS on the SQL Server, as the same instance DBackup connects to. If it fails here, no DBackup setting will fix it. + +```sql +BACKUP DATABASE [master] TO DISK = N'\\synology-nas\sql-backup\permission-test.bak' WITH INIT +``` +::: + +### Restoring to a Windows server + +Restore uses the same route the backup used and needs no extra setting. When the target database name differs from the one in the backup, DBackup relocates the data and log files into the instance's own default directories, which it reads from the server. On a server too old to report them, SQL Server 2008 R2 and earlier, the files stay in the directory the backup records. + ## Setting Up a Backup User Create a dedicated login with backup permissions: diff --git a/scripts/setup-dev-debian.sh b/scripts/setup-dev-debian.sh index 284a7bd7..62fe4cef 100755 --- a/scripts/setup-dev-debian.sh +++ b/scripts/setup-dev-debian.sh @@ -149,6 +149,51 @@ redis-cli --version && info "Redis CLI installed โœ“" || warn "Redis CLI check f info "Installing additional tools (SSH, rsync, smbclient)..." apt-get install -y -qq openssh-client sshpass rsync smbclient openssl zip > /dev/null +# ------------------------------------------------------------------- +# SqlPackage - BACPAC export/import for the Azure SQL Database adapter +# +# Installed as a dotnet tool rather than from Microsoft's standalone zip, which +# is published for linux-x64 only. The tool is portable IL and works on arm64, +# which is the same route the container image takes. +# ------------------------------------------------------------------- +info "Installing SqlPackage (Azure SQL Database adapter)..." +if ! command -v dotnet &>/dev/null; then + apt-get install -y -qq libicu-dev &>/dev/null || apt-get install -y -qq libicu72 &>/dev/null || true + curl -fsSL https://dot.net/v1/dotnet-install.sh -o /tmp/dotnet-install.sh + bash /tmp/dotnet-install.sh --channel 10.0 --install-dir /usr/share/dotnet --no-path + ln -sf /usr/share/dotnet/dotnet /usr/local/bin/dotnet + rm -f /tmp/dotnet-install.sh +fi +if command -v dotnet &>/dev/null; then + SQLPKG_TMP="$(mktemp -d)" + if dotnet tool install --tool-path "$SQLPKG_TMP" microsoft.sqlpackage &>/dev/null; then + # The package ships several target frameworks side by side, and the first one + # `find` returns is net8.0, whose launcher demands a .NET 8 runtime this script + # does not install. Match the runtime that is actually present. + SQLPKG_MAJOR="$(dotnet --list-runtimes | awk '/Microsoft.NETCore.App/ {print $2}' | cut -d. -f1 | sort -n | tail -1)" + SQLPKG_PAYLOAD="$(find "$SQLPKG_TMP/.store" -type d -path "*/tools/net${SQLPKG_MAJOR}.0/any" | head -1)" + [ -z "$SQLPKG_PAYLOAD" ] && SQLPKG_PAYLOAD="$(find "$SQLPKG_TMP/.store" -type d -path '*/tools/net*/any' | sort -V | tail -1)" + + if [ -n "$SQLPKG_PAYLOAD" ]; then + rm -rf /usr/local/share/sqlpackage + mkdir -p /usr/local/share/sqlpackage + cp -a "$SQLPKG_PAYLOAD"/. /usr/local/share/sqlpackage/ + + # A wrapper naming the runtime outright, the same shape the Dockerfile and + # the macOS script use. The apphost shim would rely on the default runtime + # probe paths, which is one more thing to differ between machines. + printf '#!/bin/sh\nexec "%s" /usr/local/share/sqlpackage/sqlpackage.dll "$@"\n' \ + "$(command -v dotnet)" > /usr/local/bin/sqlpackage + chmod +x /usr/local/bin/sqlpackage + else + echo -e " ${RED}Could not locate the SqlPackage payload - skipping.${NC}" + fi + else + echo -e " ${RED}SqlPackage install failed - the Azure SQL Database adapter will not work locally.${NC}" + fi + rm -rf "$SQLPKG_TMP" +fi + # ------------------------------------------------------------------- # Summary # ------------------------------------------------------------------- @@ -157,7 +202,7 @@ info "=========================================" info " DBackup Dev Dependencies โ€” Summary" info "=========================================" echo "" -for cmd in mysql mysqldump mongodump mongorestore mongosh sqlite3 redis-cli pg_dump psql rsync smbclient sshpass; do +for cmd in mysql mysqldump mongodump mongorestore mongosh sqlite3 redis-cli pg_dump psql rsync smbclient sshpass sqlpackage; do if command -v "$cmd" &>/dev/null; then echo -e " ${GREEN}โœ“${NC} $cmd ($(command -v "$cmd"))" else @@ -173,4 +218,5 @@ for ver in "${PG_VERSIONS[@]}"; do fi done echo "" -info "Done. MSSQL uses the Node.js mssql driver โ€” no binary needed." +info "Done. MSSQL uses the Node.js mssql driver - no binary needed." +info "Azure SQL Database additionally needs sqlpackage, listed above." diff --git a/scripts/setup-dev-macos.sh b/scripts/setup-dev-macos.sh index dc69cbab..35f94275 100755 --- a/scripts/setup-dev-macos.sh +++ b/scripts/setup-dev-macos.sh @@ -101,6 +101,58 @@ brew install rsync echo -e "${GREEN}Installing sshpass (for Rsync password authentication)...${NC}" brew install hudochenkov/sshpass/sshpass || echo -e "${YELLOW}sshpass install failed - password auth for rsync will not work. Use SSH keys instead.${NC}" +echo -e "${GREEN}Installing SqlPackage (BACPAC export/import for the Azure SQL Database adapter)...${NC}" +echo -e "${YELLOW}Microsoft's standalone macOS download is x64-only. The dotnet tool is portable IL${NC}" +echo -e "${YELLOW}and runs natively on Apple Silicon, which is the route the container image takes too.${NC}" +if ! command -v dotnet &> /dev/null; then + brew install dotnet +fi +if command -v dotnet &> /dev/null; then + SQLPKG_PREFIX="$(brew --prefix)" + SQLPKG_SHARE="$SQLPKG_PREFIX/share/sqlpackage" + SQLPKG_TMP="$(mktemp -d)" + + # Installed into a temp path and then relocated, rather than left in + # ~/.dotnet/tools. That directory is on nobody's PATH by default, and a dev + # server started from an editor inherits its environment at launch, so the + # adapter reported "sqlpackage was not found" even after a correct install. + # $(brew --prefix)/bin is already on PATH for anyone with Homebrew. + if dotnet tool install --tool-path "$SQLPKG_TMP" microsoft.sqlpackage > /dev/null 2>&1; then + # The package ships several target frameworks side by side. Picking the first + # one `find` returns lands on net8.0, whose launcher then demands a .NET 8 + # runtime that a Homebrew install of dotnet 10 does not have. Match the + # installed runtime instead, and fall back to the newest build on offer. + SQLPKG_MAJOR="$(dotnet --list-runtimes | awk '/Microsoft.NETCore.App/ {print $2}' | cut -d. -f1 | sort -n | tail -1)" + SQLPKG_PAYLOAD="$(find "$SQLPKG_TMP/.store" -type d -path "*/tools/net${SQLPKG_MAJOR}.0/any" | head -1)" + if [ -z "$SQLPKG_PAYLOAD" ]; then + SQLPKG_PAYLOAD="$(find "$SQLPKG_TMP/.store" -type d -path '*/tools/net*/any' | sort -V | tail -1)" + fi + if [ -n "$SQLPKG_PAYLOAD" ]; then + rm -rf "$SQLPKG_SHARE" + mkdir -p "$SQLPKG_SHARE" + cp -a "$SQLPKG_PAYLOAD"/. "$SQLPKG_SHARE/" + + # A wrapper rather than the apphost shim. The shim probes the default + # /usr/local/share/dotnet for a runtime that Homebrew keeps under its own + # prefix, and fails with "Download the .NET runtime" - which reads as a + # missing install rather than a missing DOTNET_ROOT. Naming the runtime + # outright removes the variable from the picture entirely. + printf '#!/bin/sh\nexec "%s" "%s/sqlpackage.dll" "$@"\n' \ + "$(command -v dotnet)" "$SQLPKG_SHARE" > "$SQLPKG_PREFIX/bin/sqlpackage" + chmod +x "$SQLPKG_PREFIX/bin/sqlpackage" + + echo -e "${GREEN}SqlPackage installed to $SQLPKG_PREFIX/bin/sqlpackage ($("$SQLPKG_PREFIX/bin/sqlpackage" /version 2>/dev/null | tail -1))${NC}" + else + echo -e "${RED}Could not locate the SqlPackage payload - skipping.${NC}" + fi + else + echo -e "${RED}SqlPackage install failed - the Azure SQL Database adapter will not work locally.${NC}" + fi + rm -rf "$SQLPKG_TMP" +else + echo -e "${RED}dotnet is unavailable - skipping SqlPackage. The Azure SQL Database adapter will not work locally.${NC}" +fi + echo -e "${GREEN}Installing generally useful tools (zip)...${NC}" brew install zip @@ -113,6 +165,8 @@ echo -e "${YELLOW}/opt/homebrew/opt/libpq/bin - if that comes first, native comp echo "" echo 'export PATH="/opt/homebrew/opt/mysql-client/bin:/opt/homebrew/opt/postgresql@18/bin:/opt/homebrew/opt/postgresql@16/bin:/opt/homebrew/opt/postgresql@14/bin:/opt/homebrew/firebird-client/bin:$PATH"' echo "" +echo -e "${GREEN}SqlPackage needs no PATH entry - it was installed into $(brew --prefix)/bin, which is already there.${NC}" +echo "" echo -e "${YELLOW}Add to ~/.zshrc permanently:${NC}" echo 'echo '\''export PATH="/opt/homebrew/opt/mysql-client/bin:/opt/homebrew/opt/postgresql@18/bin:/opt/homebrew/opt/postgresql@16/bin:/opt/homebrew/opt/postgresql@14/bin:/opt/homebrew/firebird-client/bin:$PATH"'\'' >> ~/.zshrc' echo 'source ~/.zshrc' diff --git a/src/app/actions/settings/settings.ts b/src/app/actions/settings/settings.ts index f18f40a1..8c8ab8a4 100644 --- a/src/app/actions/settings/settings.ts +++ b/src/app/actions/settings/settings.ts @@ -8,6 +8,7 @@ import { PERMISSIONS } from "@/lib/auth/permissions"; import { logger } from "@/lib/logging/logger"; import { wrapError } from "@/lib/logging/errors"; import { scheduler } from "@/lib/server/scheduler"; +import { isValidTimezone } from "@/lib/utils"; import { STUCK_TIMEOUT_SETTING } from "@/services/system/stuck-execution-service"; const log = logger.child({ action: "settings" }); @@ -25,10 +26,7 @@ const settingsSchema = z.object({ checkForUpdates: z.boolean().optional(), showQuickSetup: z.boolean().optional(), systemTimezone: z.string() - .refine((tz) => { - try { return Intl.supportedValuesOf('timeZone').includes(tz) || tz === 'UTC'; } - catch { return false; } - }, { message: "Invalid IANA timezone" }) + .refine(isValidTimezone, { message: "Invalid IANA timezone" }) .optional(), filenamePattern: z.string().min(1).optional(), instanceName: z.string().max(50).optional(), diff --git a/src/app/api/storage/[id]/analyze/route.ts b/src/app/api/storage/[id]/analyze/route.ts index 7ddc8f35..9c639e1a 100644 --- a/src/app/api/storage/[id]/analyze/route.ts +++ b/src/app/api/storage/[id]/analyze/route.ts @@ -110,7 +110,7 @@ export async function POST(req: NextRequest, props: { params: Promise<{ id: stri } // For server-based adapters (not sqlite) with empty names, // use the source type to signal the frontend that this is a DB restore - const serverAdapters = ['mysql', 'mariadb', 'postgres', 'mongodb', 'mssql', 'redis', 'valkey']; + const serverAdapters = ['mysql', 'mariadb', 'postgres', 'mongodb', 'mssql', 'azure-sql', 'redis', 'valkey', 'firebird']; if (meta.sourceType && serverAdapters.includes(meta.sourceType.toLowerCase())) { return NextResponse.json({ databases: [], sourceType: meta.sourceType }); } diff --git a/src/app/dashboard/storage/restore/restore-client.tsx b/src/app/dashboard/storage/restore/restore-client.tsx index fcca622e..e3b5a8be 100644 --- a/src/app/dashboard/storage/restore/restore-client.tsx +++ b/src/app/dashboard/storage/restore/restore-client.tsx @@ -200,7 +200,7 @@ export function RestoreClient({ canManageVault = false }: RestoreClientProps) { const isSystemConfig = file?.sourceType === 'SYSTEM'; - const SERVER_ADAPTERS = ['mysql', 'mariadb', 'postgres', 'mongodb', 'mssql', 'redis', 'valkey', 'firebird']; + const SERVER_ADAPTERS = ['mysql', 'mariadb', 'postgres', 'mongodb', 'mssql', 'azure-sql', 'redis', 'valkey', 'firebird']; const resolvedSourceType = backupSourceType || file?.sourceType || ''; const isServerAdapter = SERVER_ADAPTERS.includes(resolvedSourceType.toLowerCase()); // Firebird's target field holds a filesystem path, not a database name - and since diff --git a/src/components/adapter/adapter-manager.tsx b/src/components/adapter/adapter-manager.tsx index 6324bfa2..fddc3dfe 100644 --- a/src/components/adapter/adapter-manager.tsx +++ b/src/components/adapter/adapter-manager.tsx @@ -173,6 +173,7 @@ export function AdapterManager({ type, title, description, canManage = true, per case 'postgres': case 'mariadb': case 'mssql': + case 'azure-sql': case 'mongodb': return {config.user}@{config.host}:{config.port}; case 'redis': diff --git a/src/components/adapter/form-constants.ts b/src/components/adapter/form-constants.ts index a01b9c01..c65807c1 100644 --- a/src/components/adapter/form-constants.ts +++ b/src/components/adapter/form-constants.ts @@ -50,6 +50,10 @@ export const PLACEHOLDERS: Record = { "postgres.port": "5432", "mongodb.port": "27017", "mssql.port": "1433", + "azure-sql.host": "myserver.database.windows.net", + "azure-sql.port": "1433", + "azure-sql.user": "backupadmin", + "azure-sql.requestTimeout": "300000", "redis.port": "6379", "valkey.port": "6379", "firebird.port": "3050", diff --git a/src/components/adapter/form-sections.tsx b/src/components/adapter/form-sections.tsx index 39d8beca..cf2868b0 100644 --- a/src/components/adapter/form-sections.tsx +++ b/src/components/adapter/form-sections.tsx @@ -25,12 +25,13 @@ import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { transferConcurrencyRange } from "@/lib/adapters/transfer-concurrency"; +import { s3UploadTuningRange, s3UploadMemoryBudget, S3_MIN_PART_SIZE_MB } from "@/lib/adapters/s3-upload-tuning"; import { Label } from "@/components/ui/label"; import { Switch } from "@/components/ui/switch"; import { RadioGroup, RadioGroupItem } from "@/components/ui/radio-group"; import { STORAGE_ROLES, type StorageRole } from "@/lib/core/storage-roles"; import { sshManagedKeys } from "@/lib/adapters/ssh-key-convention"; -import { cn } from "@/lib/utils"; +import { cn, formatBytes } from "@/lib/utils"; import { AlertTriangle, Check, ChevronDown, FolderOpen, Loader2 } from "lucide-react"; import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; import { Alert, AlertDescription } from "@/components/ui/alert"; @@ -243,6 +244,101 @@ function TransferConcurrencyField({ ); } +/** + * How one archive is split across parallel connections on the way to an object store. + * + * Separate from `TransferConcurrencyField` above, which counts whole files and only means + * something for a directory source. A backup destination receives one archive per run, so the + * parallelism has to happen inside that single upload instead. + * + * The two inputs sit together and show their product because they are meaningless apart: the + * peak memory of an upload is the parts in flight times their size, so the same step in + * parallelism costs eight times as much on 64 MB parts as on 8 MB ones. + * + * The part size is asked for as a maximum rather than a value, because the size that performs + * depends on how large the archive turns out to be and the archive differs every run. What the + * user can actually decide is how much memory to spend, which is what a ceiling expresses. + * `resolveS3UploadTuning` picks the largest size at or below it that still keeps every + * connection busy. + */ +function S3UploadTuningFields({ + adapterId, + concurrency, + partSizeMb, + onConcurrencyChange, + onPartSizeChange, +}: { + adapterId: string; + concurrency: number | undefined; + partSizeMb: number | undefined; + onConcurrencyChange: (value: number) => void; + onPartSizeChange: (value: number) => void; +}) { + const range = s3UploadTuningRange(adapterId); + if (!range) return null; + + // Clamped for display, not just on input: a ceiling lowered in a later version leaves + // stored values above it, and the runtime clamps them anyway. Showing the stored number + // would claim a parallelism the connection will never actually use. + const currentConcurrency = Math.min(range.concurrency.max, concurrency ?? range.concurrency.default); + const currentPartSize = Math.min(range.partSizeMb.max, Math.max(S3_MIN_PART_SIZE_MB, partSizeMb ?? range.partSizeMb.default)); + + return ( +
+
+ +

+ A backup is uploaded as several parts at once. More parts use more of a fast + link, at the cost of memory while the upload runs. +

+
+
+
+ + { + const parsed = parseInt(e.target.value, 10); + if (!Number.isFinite(parsed)) return; + onConcurrencyChange(Math.min(range.concurrency.max, Math.max(1, parsed))); + }} + /> +
+
+ + { + const parsed = parseInt(e.target.value, 10); + if (!Number.isFinite(parsed)) return; + onPartSizeChange(Math.min(range.partSizeMb.max, Math.max(S3_MIN_PART_SIZE_MB, parsed))); + }} + /> +
+
+

+ Uses up to {formatBytes(s3UploadMemoryBudget(currentConcurrency, currentPartSize))} of + memory per upload. Smaller backups automatically use smaller parts, so that every + connection still gets one. Defaults are {range.concurrency.default} parts + of {range.partSizeMb.default} MB, up to {range.concurrency.max} parts + of {range.partSizeMb.max} MB. +

+
+ ); +} + function DisableVerificationSwitch({ disabled, onChange, @@ -1127,6 +1223,19 @@ export function StorageFormContent({ onChange={(v) => setValue("config.maxConcurrentFiles", v, { shouldDirty: true })} /> )} + {/* The mirror image of the field above, and the reason each is limited to one + role. Reading a directory means many files and one connection each, so + what matters there is how many files run at once. Writing a backup means + one archive, so the only parallelism left is inside that upload. */} + {storageRole !== STORAGE_ROLES.SOURCE && ( + setValue("config.uploadConcurrency", v, { shouldDirty: true })} + onPartSizeChange={(v) => setValue("config.uploadPartSizeMb", v, { shouldDirty: true })} + /> + )} {/* Only for a directory source: a snapshot of the place backups are written to serves no purpose. */} {supportsSnapshots && storageRole === STORAGE_ROLES.SOURCE && ( diff --git a/src/components/adapter/utils.ts b/src/components/adapter/utils.ts index 4ba09c3d..4e411231 100644 --- a/src/components/adapter/utils.ts +++ b/src/components/adapter/utils.ts @@ -28,6 +28,7 @@ import discordIcon from "@iconify-icons/logos/discord-icon"; import slackIcon from "@iconify-icons/logos/slack-icon"; import teamsIcon from "@iconify-icons/logos/microsoft-teams"; import telegramIcon from "@iconify-icons/logos/telegram"; +import azureIcon from "@iconify-icons/logos/azure"; // - Simple Icons (fallback for brands not in SVG Logos) - import mssqlIcon from "@iconify-icons/simple-icons/microsoftsqlserver"; @@ -74,6 +75,8 @@ const ADAPTER_ICON_MAP: Record = { "redis": redisIcon, "valkey": valkeyIcon, "mssql": mssqlIcon, + // logos:azure is multi-colour, so no ADAPTER_COLOR_MAP entry is needed. + "azure-sql": azureIcon, "firebird": firebirdIcon, // Storage - Local diff --git a/src/components/dashboard/jobs/job-form.tsx b/src/components/dashboard/jobs/job-form.tsx index 7995ca1c..1e88d152 100644 --- a/src/components/dashboard/jobs/job-form.tsx +++ b/src/components/dashboard/jobs/job-form.tsx @@ -63,22 +63,12 @@ const COMBINABLE_DB_ADAPTERS = ["mysql", "mariadb", "postgres", "mongodb", "fire /** Which kind of source(s) this job backs up - purely client-side UI state, not persisted. */ type SourceMode = "db" | "dirs" | "both"; -const retentionSchema = z.object({ - mode: z.enum(["NONE", "SIMPLE", "SMART"]), - simple: z.object({ - keepCount: z.coerce.number().min(1).default(10) - }).optional(), - smart: z.object({ - daily: z.coerce.number().min(0).default(7), - weekly: z.coerce.number().min(0).default(4), - monthly: z.coerce.number().min(0).default(12), - yearly: z.coerce.number().min(0).default(2), - }).optional() -}); - const destinationSchema = z.object({ configId: z.string().min(1, "Destination is required"), - retention: retentionSchema, + // Legacy inline retention, round-tripped untouched. Destinations are configured through + // a RetentionPolicy template now, and this form has no editor for the inline JSON, so a + // typed z.object() here would only strip fields it does not know about on every save. + retention: z.record(z.string(), z.unknown()), retentionPolicyId: z.string().optional(), }); @@ -1959,75 +1949,3 @@ function DirectoryBrowseDialog({ open, onOpenChange, configId, adapterName, init ); } -// --- Retention Config Component (reusable per destination) --- - -function _RetentionConfig({ form, prefix }: { form: any; prefix: string }) { - const mode = form.watch(`${prefix}.mode`); - - return ( -
- ( - - - Keep All - Simple - Smart (GFS) - - - )} - /> - - {mode === "NONE" && ( -

All backups kept indefinitely.

- )} - - {mode === "SIMPLE" && ( - ( - -
- - field.onChange(parseInt(e.target.value))} className="w-20 h-8" /> - - newest backups -
- - - )} - /> - )} - - {mode === "SMART" && ( -
- {(["daily", "weekly", "monthly", "yearly"] as const).map(period => ( - ( - - {period} - - field.onChange(parseInt(e.target.value))} - className="h-8" - /> - - - )} - /> - ))} -
- )} -
- ); -} diff --git a/src/components/settings/system-settings-form.tsx b/src/components/settings/system-settings-form.tsx index e418a0e7..e66f892f 100644 --- a/src/components/settings/system-settings-form.tsx +++ b/src/components/settings/system-settings-form.tsx @@ -3,7 +3,7 @@ import { zodResolver } from "@hookform/resolvers/zod" import { useForm, useWatch } from "react-hook-form" import * as z from "zod" -import { useState } from "react" +import { useMemo, useState } from "react" import { Form, FormControl, @@ -71,7 +71,14 @@ interface SystemSettingsFormProps { export function SystemSettingsForm({ initialMaxConcurrentJobs, initialStuckTimeoutMinutes = 360, initialDisablePasskeyLogin, initialSessionDuration = 604800, initialAuditLogRetentionDays = 90, initialStorageSnapshotRetentionDays = 90, initialNotificationLogRetentionDays = 90, initialCheckForUpdates = true, initialShowQuickSetup = false, initialSystemTimezone = "UTC", initialFilenamePattern = "{name}_yyyy-MM-dd_HH-mm-ss", initialInstanceName = "", emailLoginDisabledByEnv = false }: SystemSettingsFormProps) { const [openTimezone, setOpenTimezone] = useState(false); - const timezones = Intl.supportedValuesOf('timeZone'); + // Browsers disagree on which IANA name is canonical, so a zone stored from one browser can be + // absent from another's list. Prepend the stored value when it is missing, otherwise it would + // be unfindable and show no checkmark even though it is the active setting. + const timezones = useMemo(() => { + const supported = Intl.supportedValuesOf('timeZone'); + const current = initialSystemTimezone || "UTC"; + return supported.includes(current) ? supported : [current, ...supported]; + }, [initialSystemTimezone]); const form = useForm>({ resolver: zodResolver(formSchema) as any, defaultValues: { diff --git a/src/components/settings/templates/retention-policy-form.tsx b/src/components/settings/templates/retention-policy-form.tsx index 2869a784..a53a7610 100644 --- a/src/components/settings/templates/retention-policy-form.tsx +++ b/src/components/settings/templates/retention-policy-form.tsx @@ -1,9 +1,17 @@ "use client"; +import { useState } from "react"; +import { Plus, X } from "lucide-react"; +import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; -import { RetentionConfiguration, RetentionMode } from "@/lib/core/retention"; +import { + DEFAULT_HOURLY_TIER, + RetentionConfiguration, + RetentionMode, +} from "@/lib/core/retention"; +import { cn } from "@/lib/utils"; interface Props { value: RetentionConfiguration; @@ -11,13 +19,19 @@ interface Props { } const DEFAULT_SIMPLE = { keepCount: 10 }; -const DEFAULT_SMART = { daily: 7, weekly: 4, monthly: 12, yearly: 2 }; +const DEFAULT_SMART = { hourly: 0, daily: 7, weekly: 4, monthly: 12, yearly: 2 }; export function RetentionPolicyForm({ value, onChange }: Props) { const mode = value.mode; const simple = value.simple ?? DEFAULT_SIMPLE; const smart = value.smart ?? DEFAULT_SMART; + // Most setups never need an hourly tier, so the field stays out of the way until it is + // asked for. A policy that already carries one opens with it visible, which is why this + // is derived from the value rather than held in state alone. + const [manuallyShown, setManuallyShown] = useState(false); + const showHourly = manuallyShown || (smart.hourly ?? 0) > 0; + function setMode(newMode: RetentionMode) { onChange({ mode: newMode, @@ -37,6 +51,16 @@ export function RetentionPolicyForm({ value, onChange }: Props) { }); } + function toggleHourly() { + if (showHourly) { + setManuallyShown(false); + setSmartField("hourly", 0); + return; + } + setManuallyShown(true); + setSmartField("hourly", DEFAULT_HOURLY_TIER); + } + return (
@@ -78,21 +102,68 @@ export function RetentionPolicyForm({ value, onChange }: Props) { )} {mode === "SMART" && ( -
- {(["daily", "weekly", "monthly", "yearly"] as const).map((period) => ( -
- - - setSmartField(period, parseInt(e.target.value) || 0) - } - className="h-8" - /> -
- ))} +
+
+ {showHourly && ( +
+ + + setSmartField("hourly", parseInt(e.target.value) || 0) + } + className="h-8" + /> +
+ )} + {(["daily", "weekly", "monthly", "yearly"] as const).map((period) => ( +
+ + + setSmartField(period, parseInt(e.target.value) || 0) + } + className="h-8" + /> +
+ ))} +
+ + + +

+ The tiers add up rather than overlap. Each one keeps that many backups on top + of what the finer tiers already cover, so hourly 24 with daily 7 reaches back + about a day of hourly slots plus 7 further days. +

)}
diff --git a/src/components/settings/templates/retention-policy-list.tsx b/src/components/settings/templates/retention-policy-list.tsx index 1388c35c..f0c82ac7 100644 --- a/src/components/settings/templates/retention-policy-list.tsx +++ b/src/components/settings/templates/retention-policy-list.tsx @@ -110,7 +110,16 @@ export function RetentionPolicyList() { return `Simple - keep ${parsed.simple?.keepCount ?? "?"} backups`; if (parsed.mode === "SMART") { const s = parsed.smart; - return `Smart GFS (${s?.daily ?? 0}/${s?.weekly ?? 0}/${s?.monthly ?? 0}/${s?.yearly ?? 0})`; + // Suffixed because a bare "24/7/4/12/2" cannot be told apart from "7/4/12/2" + // once an hourly tier can be present. Hourly only shows when it is in use. + const tiers = [ + ...(s?.hourly ? [`${s.hourly}h`] : []), + `${s?.daily ?? 0}d`, + `${s?.weekly ?? 0}w`, + `${s?.monthly ?? 0}m`, + `${s?.yearly ?? 0}y`, + ]; + return `Smart GFS (${tiers.join("/")})`; } } catch { // ignore diff --git a/src/lib/adapters/CLAUDE.md b/src/lib/adapters/CLAUDE.md index ca7514f1..8a89d7c8 100644 --- a/src/lib/adapters/CLAUDE.md +++ b/src/lib/adapters/CLAUDE.md @@ -50,6 +50,7 @@ Rules: - **Build raw argv arrays. Never escape anything.** `shellEscape` is internal to `SshHost`. An adapter that escapes produces a double-escaped argument that fails only over SSH, only at runtime. - **`exec` returns a `code`, it does not throw on non-zero.** Check `result.code !== 0` explicitly. Code relying on a rejected promise silently stops iterating instead. - **Secrets go in `options.env`, never in argv.** `SshHost` renders them into an `export` prefix, which keeps them out of the process table and out of OOM kill reports. + - **One named exception: `database/azure-sql/exporter/sqlpackage.ts`.** SqlPackage has no environment route and rejects `/SourceConnectionString:@file`, so the connection string has to be an argument. The exception holds only because that adapter has no SSH mode at all - its schema carries no `connectionMode`, so `standardTransport` always returns a DirectHost, the argv array never reaches a shell, and the exposure is the process table of the container that already holds the password in memory. Adding an SSH mode there invalidates this and the secret handling has to be reworked first. A unit test asserts the argv so the exception cannot be quietly removed or quietly widened. - **Never call `spawn` / `execFile` directly**, and never open your own connection. Use `host.spawn`, `host.exec`, `host.connect`, `host.forwardPort`. - **File movement uses host primitives**: `withTempFile`, `stageInput`, `captureOutput`, `putFile`, `getFile`. They are no-ops in direct mode and SFTP transfers over SSH, so one call covers both. - Spread `...sshFields` into the config schema. If the field layout differs, declare a `transport` resolver on the adapter instead of reading `connectionMode` in adapter code. **Zod defaults do not run at runtime** (`resolveAdapterConfig` returns decrypted JSON), so default `undefined` to direct in code. diff --git a/src/lib/adapters/database/azure-sql/browser.ts b/src/lib/adapters/database/azure-sql/browser.ts new file mode 100644 index 00000000..9250e654 --- /dev/null +++ b/src/lib/adapters/database/azure-sql/browser.ts @@ -0,0 +1,153 @@ +import sql from "mssql"; +import type { ExecutionHost } from "@/lib/transport"; +import type { AzureSQLConfig } from "@/lib/adapters/definitions"; +import type { TableInfo, ColumnInfo, TableDataOptions, TableDataResult } from "@/lib/core/interfaces"; +import { withPool } from "./pool"; + +/** + * Table browsing on Azure SQL Database. + * + * Structurally the MSSQL browser with every three-part name removed. Azure SQL + * Database rejects `[db].schema.object` outright ("Reference to database and/or + * server name in ... is not supported in this version of SQL Server"), so the + * database is selected by connecting to it instead. `withPool(..., { database })` + * is what makes that a one-word difference at each call site. + * + * Every query below is therefore two-part at most. A three-part name reintroduced + * here would work in no environment at all, which is why the tests assert on it. + */ + +/** Sanitize an identifier for bracket-quoting. */ +function escapeIdentifier(name: string): string { + return name.replace(/]/g, "]]").replace(/\0/g, ""); +} + +/** Sanitize a value for use in a single-quoted SQL string literal. */ +function escapeStringLiteral(name: string): string { + return name.replace(/'/g, "''").replace(/\0/g, ""); +} + +export async function getTables( + config: AzureSQLConfig, + database: string, + host: ExecutionHost, +): Promise { + return withPool(config, host, async (pool) => { + const result = await pool.request().query(` + SELECT + t.TABLE_SCHEMA AS schema_name, + t.TABLE_NAME AS name, + t.TABLE_TYPE AS table_type, + COALESCE(SUM(p.rows), 0) AS row_count, + COALESCE(SUM(CAST(a.total_pages AS BIGINT)) * 8 * 1024, 0) AS size_bytes + FROM INFORMATION_SCHEMA.TABLES t + LEFT JOIN sys.tables st ON st.name = t.TABLE_NAME AND st.schema_id = SCHEMA_ID(t.TABLE_SCHEMA) + LEFT JOIN sys.indexes i ON i.object_id = st.object_id AND i.type <= 1 + LEFT JOIN sys.partitions p ON p.object_id = st.object_id AND p.index_id = i.index_id + LEFT JOIN sys.allocation_units a ON a.container_id = p.partition_id + GROUP BY t.TABLE_SCHEMA, t.TABLE_NAME, t.TABLE_TYPE + ORDER BY t.TABLE_SCHEMA, t.TABLE_NAME + `); + + return result.recordset.map((row: Record) => ({ + name: row.schema_name !== "dbo" ? `${row.schema_name}.${row.name}` : String(row.name), + type: (row.table_type === "VIEW" ? "view" : "table") as TableInfo["type"], + rowCount: Number(row.row_count) || 0, + sizeInBytes: Number(row.size_bytes) || 0, + })); + }, { database }); +} + +export async function getTableData( + config: AzureSQLConfig, + options: TableDataOptions, + host: ExecutionHost, +): Promise { + const { database, table, page, pageSize, sortBy, sortDir, search, searchColumn, matchMode } = options; + const offset = (page - 1) * pageSize; + + // Tables in schemas other than dbo are stored as "schema.tableName". + let tableSchema = "dbo"; + let tableName = table; + if (table.includes(".")) { + const dotIndex = table.indexOf("."); + tableSchema = table.substring(0, dotIndex); + tableName = table.substring(dotIndex + 1); + } + + const schemaId = escapeIdentifier(tableSchema); + const tblId = escapeIdentifier(tableName); + const schemaLiteral = escapeStringLiteral(tableSchema); + const tblLiteral = escapeStringLiteral(tableName); + + const sortColExpr = sortBy + ? `[${escapeIdentifier(sortBy)}] ${sortDir === "desc" ? "DESC" : "ASC"}` + : "(SELECT NULL)"; + + const searchActive = !!(search && searchColumn); + const searchTermValue = searchActive + ? matchMode === "starts" ? `${search}%` + : matchMode === "ends" ? `%${search}` + : matchMode === "equals" ? search! + : `%${search}%` + : undefined; + const whereClause = searchActive + ? matchMode === "equals" + ? ` WHERE CAST([${escapeIdentifier(searchColumn!)}] AS NVARCHAR(MAX)) = @searchTerm` + : ` WHERE CAST([${escapeIdentifier(searchColumn!)}] AS NVARCHAR(MAX)) LIKE @searchTerm` + : ""; + + return withPool(config, host, async (pool) => { + const colReq = pool.request(); + const countReq = pool.request(); + const dataReq = pool.request(); + + if (searchActive) { + countReq.input("searchTerm", sql.NVarChar, searchTermValue); + dataReq.input("searchTerm", sql.NVarChar, searchTermValue); + } + + const [colResult, countResult, dataResult] = await Promise.all([ + colReq.query(` + SELECT COLUMN_NAME, DATA_TYPE, IS_NULLABLE, + CASE WHEN COLUMN_NAME IN ( + SELECT kcu.COLUMN_NAME FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc + JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE kcu + ON tc.CONSTRAINT_NAME = kcu.CONSTRAINT_NAME AND tc.TABLE_SCHEMA = kcu.TABLE_SCHEMA + WHERE tc.CONSTRAINT_TYPE = 'PRIMARY KEY' AND tc.TABLE_NAME = '${tblLiteral}' + ) THEN 'PRI' ELSE '' END AS COLUMN_KEY, + COLUMN_DEFAULT + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_NAME = '${tblLiteral}' AND TABLE_SCHEMA = '${schemaLiteral}' + ORDER BY ORDINAL_POSITION + `), + countReq.query(`SELECT COUNT(*) AS total FROM [${schemaId}].[${tblId}]${whereClause}`), + dataReq.query(` + SELECT * FROM [${schemaId}].[${tblId}]${whereClause} + ORDER BY ${sortColExpr} + OFFSET ${offset} ROWS FETCH NEXT ${pageSize} ROWS ONLY + `), + ]); + + const columns: ColumnInfo[] = colResult.recordset.map((row: Record) => ({ + name: String(row.COLUMN_NAME), + dataType: String(row.DATA_TYPE), + nullable: row.IS_NULLABLE === "YES", + primaryKey: row.COLUMN_KEY === "PRI", + defaultValue: (row.COLUMN_DEFAULT as string | undefined) ?? undefined, + })); + + const totalCount = Number(countResult.recordset[0]?.total) || 0; + + const rows: Record[] = dataResult.recordset.map((row: Record) => { + const record: Record = {}; + for (const col of columns) { + const val = row[col.name]; + record[col.name] = val === undefined ? null : val; + } + return record; + }); + + return { rows, totalCount, columns }; + }, { database }); +} diff --git a/src/lib/adapters/database/azure-sql/catalog.ts b/src/lib/adapters/database/azure-sql/catalog.ts new file mode 100644 index 00000000..88d52ea4 --- /dev/null +++ b/src/lib/adapters/database/azure-sql/catalog.ts @@ -0,0 +1,73 @@ +import type { ExecutionHost } from "@/lib/transport"; +import type { AzureSQLConfig } from "@/lib/adapters/definitions"; +import type { DatabaseInfo } from "@/lib/core/interfaces"; +import { logger } from "@/lib/logging/logger"; +import { wrapError } from "@/lib/logging/errors"; +import { mapWithConcurrency } from "@/lib/concurrency"; +import { withPool } from "./pool"; +import { getDatabases } from "./connection"; + +const log = logger.child({ adapter: "azure-sql" }); + +/** + * One TDS connection per database, so the explorer must not open dozens at once. + * Azure counts concurrent sessions against the service tier, and a Basic database + * allows very few. + */ +const STATS_CONCURRENCY = 8; + +/** + * Size and table count per database. + * + * The MSSQL adapter answers this with one query against `sys.master_files` plus + * cross-database `[db].sys.tables` lookups. Neither works here: `sys.master_files` + * is server scoped and does not exist on Azure SQL Database, and three-part names + * are rejected outright. Every database therefore needs its own connection, which + * is why the fan-out is bounded. + * + * A database that cannot be read degrades to a name with no size rather than + * failing the call. Letting one throw would reproduce exactly the bug this adapter + * exists downstream of, where a single missing catalog view took out the whole + * Database Explorer page. + */ +export async function getDatabasesWithStats( + config: AzureSQLConfig, + host: ExecutionHost, +): Promise { + const names = await getDatabases(config, host); + if (names.length === 0) return []; + + return mapWithConcurrency(names, STATS_CONCURRENCY, async (name): Promise => { + try { + return await withPool( + config, + host, + async (pool) => { + const [sizeResult, tableResult] = await Promise.all([ + // type = 0 is the data files. The log is excluded deliberately: a + // BACPAC never contains it, so counting it would overstate what a + // backup of this database is going to cost. + pool.request().query(` + SELECT SUM(CAST(size AS BIGINT)) * 8 * 1024 AS size_bytes + FROM sys.database_files + WHERE type = 0 + `), + pool.request().query(`SELECT COUNT(*) AS cnt FROM sys.tables`), + ]); + + const sizeBytes = sizeResult.recordset[0]?.size_bytes; + + return { + name, + sizeInBytes: sizeBytes != null ? Number(sizeBytes) : undefined, + tableCount: Number(tableResult.recordset[0]?.cnt) || 0, + }; + }, + { database: name }, + ); + } catch (error: unknown) { + log.warn("Could not read database details", { database: name }, wrapError(error)); + return { name, tableCount: 0 }; + } + }); +} diff --git a/src/lib/adapters/database/azure-sql/connection.ts b/src/lib/adapters/database/azure-sql/connection.ts new file mode 100644 index 00000000..1e66ebe2 --- /dev/null +++ b/src/lib/adapters/database/azure-sql/connection.ts @@ -0,0 +1,127 @@ +import type { ExecutionHost } from "@/lib/transport"; +import type { AzureSQLConfig } from "@/lib/adapters/definitions"; +import { logger } from "@/lib/logging/logger"; +import { wrapError } from "@/lib/logging/errors"; +import { withPool } from "./pool"; +import { resolveExporter } from "./exporter"; + +const log = logger.child({ adapter: "azure-sql" }); + +/** Azure SQL Database. Every other value belongs to a different product. */ +const ENGINE_EDITION_AZURE_SQL_DATABASE = 5; + +/** + * What to tell someone who pointed this adapter at something else. + * + * Worth being specific about. The three engines below all answer on port 1433 with + * a TDS handshake and look identical until the first catalog query, so "connection + * failed" would send people looking at firewalls. + */ +function describeWrongEngine(engineEdition: number): string { + switch (engineEdition) { + case 8: + return "This server is Azure SQL Managed Instance, not Azure SQL Database. Managed Instance is not supported by DBackup: it only accepts BACKUP DATABASE as TO URL against Azure Blob Storage, which neither adapter implements."; + case 6: + case 11: + return "This server is Azure Synapse Analytics, which is not supported."; + case 9: + return "This server is Azure SQL Edge. Use the Microsoft SQL Server source type, which backs it up natively."; + default: + return "This server is a regular SQL Server instance, not Azure SQL Database. Use the Microsoft SQL Server source type, which produces a native .bak and is the better backup for it."; + } +} + +/** + * Verify the connection, the engine, and that a BACPAC can actually be produced. + * + * The exporter probe is part of the connection test on purpose. A missing + * SqlPackage is not a connection problem, but discovering it here costs one click + * and discovering it later costs a failed scheduled run. + */ +export async function test( + config: AzureSQLConfig, + host?: ExecutionHost, +): Promise<{ success: boolean; message: string; version?: string; edition?: string }> { + try { + const result = await withPool(config, host!, (pool) => pool.request().query(` + SELECT + SERVERPROPERTY('ProductVersion') AS ProductVersion, + SERVERPROPERTY('EngineEdition') AS EngineEdition, + DATABASEPROPERTYEX(DB_NAME(), 'ServiceObjective') AS ServiceObjective + `)); + + const row = result.recordset[0] ?? {}; + const engineEdition = Number(row.EngineEdition) || 0; + + if (engineEdition !== ENGINE_EDITION_AZURE_SQL_DATABASE) { + return { success: false, message: describeWrongEngine(engineEdition) }; + } + + // Azure SQL Database has reported 12.0.2000 for years regardless of the + // engine actually running. Surfaced because the version history column + // expects something, never used to pick behaviour. + const productVersion = String(row.ProductVersion || ""); + const version = /^(\d+\.\d+\.\d+)/.exec(productVersion)?.[1] || productVersion; + const tier = row.ServiceObjective ? ` ${row.ServiceObjective}` : ""; + + const probe = await resolveExporter().probe(config, host!); + if (!probe.ok) { + return { + success: false, + message: `Connected to Azure SQL Database, but backups cannot run: ${probe.detail}`, + version, + edition: "Azure SQL Database", + }; + } + + return { + success: true, + message: `Connection successful (Azure SQL Database${tier})`, + version, + edition: "Azure SQL Database", + }; + } catch (error: unknown) { + const message = error instanceof Error ? error.message : String(error); + + if (message.includes("Login failed")) { + return { success: false, message: "Login failed. Check the user name and password." }; + } + if (message.includes("ETIMEOUT") || message.includes("ECONNREFUSED")) { + return { + success: false, + message: "Could not reach the server. Check the server name, and that a firewall rule allows this machine's IP address.", + }; + } + if (message.includes("not allowed to access")) { + return { + success: false, + message: "The server rejected this client. Add a firewall rule for this machine's IP address in the Azure portal.", + }; + } + + return { success: false, message: `Connection failed: ${message}` }; + } +} + +/** + * User databases on the logical server. + * + * The MSSQL adapter filters on `database_id > 4` to skip the four system + * databases. Azure SQL Database has only `master`, and its user databases get ids + * assigned per server with no guarantee about the range, so the filter is by name. + */ +export async function getDatabases(config: AzureSQLConfig, host: ExecutionHost): Promise { + try { + const result = await withPool(config, host, (pool) => pool.request().query(` + SELECT name + FROM sys.databases + WHERE name <> 'master' AND state = 0 + ORDER BY name + `)); + + return result.recordset.map((row: { name: string }) => row.name); + } catch (error: unknown) { + log.error("Failed to list databases", {}, wrapError(error)); + return []; + } +} diff --git a/src/lib/adapters/database/azure-sql/dump.ts b/src/lib/adapters/database/azure-sql/dump.ts new file mode 100644 index 00000000..e2bc9711 --- /dev/null +++ b/src/lib/adapters/database/azure-sql/dump.ts @@ -0,0 +1,152 @@ +import fs from "fs/promises"; +import path from "path"; +import type { ExecutionHost } from "@/lib/transport"; +import type { BackupResult } from "@/lib/core/interfaces"; +import type { LogLevel, LogType } from "@/lib/core/logs"; +import type { AzureSQLConfig } from "@/lib/adapters/definitions"; +import { formatBytes } from "@/lib/utils"; +import { createMultiDbTar, createTempDir, cleanupTempDir } from "../common/tar-utils"; +import type { TarFileEntry } from "../common/types"; +import { getDatabases } from "./connection"; +import { resolveExporter } from "./exporter"; + +/** + * A BACPAC of a live database is not a point-in-time copy. + * + * Microsoft is explicit that an export running against a database being written to + * can produce a package that is transactionally and referentially inconsistent, and + * that such a package can fail to import later. DBackup exports directly rather + * than from a copy, so this is stated in the run log of every backup, before the + * export starts rather than after it succeeds. A line that appears only in the + * documentation is a line nobody reads until they have already lost something. + */ +const CONSISTENCY_NOTICE = + "A BACPAC export is not transactionally consistent while the database is being written to. " + + "For a guaranteed-consistent backup, quiesce writes for the duration, or export from a copy made with CREATE DATABASE ... AS COPY OF."; + +/** + * Export one or more databases to BACPAC. + * + * A single database lands as a plain .bacpac at destinationPath. Several are packed + * into the shared multi-database TAR with a real manifest, which is what lets the + * runner rename the file and record the database names without opening it. + */ +export async function dump( + config: AzureSQLConfig, + destinationPath: string, + host: ExecutionHost, + onLog?: (msg: string, level?: LogLevel, type?: LogType, details?: string) => void, + _onProgress?: (percentage: number) => void, +): Promise { + const startedAt = new Date(); + const logs: string[] = []; + const log = (msg: string, level: LogLevel = "info", type: LogType = "general", details?: string) => { + logs.push(msg); + onLog?.(msg, level, type, details); + }; + + try { + const databases = await resolveDatabases(config, host, log); + const exporter = resolveExporter(); + + log(CONSISTENCY_NOTICE, "warning"); + + if (databases.length === 1) { + // captureOutput is a no-op on the DirectHost this adapter always + // resolves to, so SqlPackage writes straight to the runner's temp file + // and the size-polling progress in 02-dump.ts sees it grow. + await host.captureOutput(destinationPath, {}, (hostPath) => + exporter.exportDatabase(config, databases[0], hostPath, host, log), + ); + } else { + await exportMany(config, databases, destinationPath, host, exporter, log); + } + + const stats = await fs.stat(destinationPath); + if (stats.size === 0) { + throw new Error("Export produced an empty file. Check the run log for SqlPackage errors."); + } + + // formatBytes rather than a hand-rolled division. A BACPAC of a small + // database is a few kilobytes, and fixed MB reported that as "0.00 MB". + log(`Backup finished successfully. Size: ${formatBytes(stats.size)}`); + + return { + success: true, + path: destinationPath, + size: stats.size, + logs, + startedAt, + completedAt: new Date(), + }; + } catch (error: unknown) { + // Not logged here. The caller turns this into a thrown "Dump failed: ..." + // that the runner reports, and logging it too would put the same failure in + // the run log twice. + return { + success: false, + logs, + error: error instanceof Error ? error.message : String(error), + startedAt, + completedAt: new Date(), + }; + } +} + +/** The job's database selection, or every user database when nothing was picked. */ +async function resolveDatabases( + config: AzureSQLConfig, + host: ExecutionHost, + log: (msg: string, level?: LogLevel) => void, +): Promise { + let databases: string[] = []; + + if (Array.isArray(config.database)) { + databases = config.database.filter((s) => s && s.trim().length > 0); + } else if (typeof config.database === "string" && config.database.includes(",")) { + databases = config.database.split(",").map((s) => s.trim()).filter((s) => s.length > 0); + } else if (typeof config.database === "string" && config.database.trim().length > 0) { + databases = [config.database.trim()]; + } + + if (databases.length === 0) { + log("No databases selected, discovering all user databases"); + databases = await getDatabases(config, host); + if (databases.length === 0) { + throw new Error("No user databases found on this server."); + } + log(`Found ${databases.length} database(s): ${databases.join(", ")}`); + } + + return databases; +} + +/** Export each database to its own BACPAC, then pack them with a manifest. */ +async function exportMany( + config: AzureSQLConfig, + databases: string[], + destinationPath: string, + host: ExecutionHost, + exporter: ReturnType, + log: (msg: string, level?: LogLevel, type?: LogType, details?: string) => void, +): Promise { + // Sequential on purpose. Concurrent exports of one Azure database multiply the + // DTU cost of the backup window, and the service tier is the user's bill. + const stagingDir = await createTempDir("azure-sql-"); + try { + const entries: TarFileEntry[] = []; + + for (const dbName of databases) { + const localPath = path.join(stagingDir, `${dbName}.bacpac`); + await host.captureOutput(localPath, {}, (hostPath) => + exporter.exportDatabase(config, dbName, hostPath, host, log), + ); + entries.push({ name: `${dbName}.bacpac`, path: localPath, dbName, format: "bacpac" }); + } + + log(`Packing ${entries.length} exports into an archive`); + await createMultiDbTar(entries, destinationPath, { sourceType: "azure-sql" }); + } finally { + await cleanupTempDir(stagingDir); + } +} diff --git a/src/lib/adapters/database/azure-sql/exporter/connection-string.ts b/src/lib/adapters/database/azure-sql/exporter/connection-string.ts new file mode 100644 index 00000000..de02ef61 --- /dev/null +++ b/src/lib/adapters/database/azure-sql/exporter/connection-string.ts @@ -0,0 +1,54 @@ +import type { AzureSQLConfig } from "@/lib/adapters/definitions"; + +/** + * The ADO.NET connection string SqlPackage is given. + * + * Its own file so the password has exactly one home and exactly one test. The + * quoting below is the whole reason: a password containing `;` silently truncates + * an unquoted connection string, and the resulting error names the wrong problem. + */ + +/** + * Quote a value for an ADO.NET connection string. + * + * Always quoted rather than only when it looks necessary. Deciding per value means + * a rule to get wrong, and the cases that need it are exactly the ones nobody + * tests: `;` ends the pair, `=` splits it, leading or trailing spaces are stripped. + * Double quotes are the delimiter, so an embedded one is doubled. + */ +function quote(value: string): string { + return `"${value.replace(/"/g, '""')}"`; +} + +/** + * Build the connection string for one database. + * + * `Encrypt=True` and `TrustServerCertificate=False` are pinned rather than + * configurable. Azure presents a real certificate on every connection, so relaxing + * either is never a legitimate setup. + */ +export function buildConnectionString(config: AzureSQLConfig, database: string): string { + const port = config.port || 1433; + + const pairs: [string, string][] = [ + ["Server", `tcp:${config.host},${port}`], + ["Initial Catalog", database], + ["User ID", config.user], + ["Password", config.password || ""], + ["Encrypt", "True"], + ["TrustServerCertificate", "False"], + ["Connection Timeout", "30"], + ]; + + return pairs.map(([key, value]) => `${key}=${quote(value)}`).join(";") + ";"; +} + +/** + * The same string with the password replaced, for logs and error messages. + * + * Never derived by regex from the real string. A redactor that has to find the + * secret can miss it, and this one cannot, because it is handed the value. + */ +export function describeConnection(config: AzureSQLConfig, database: string): string { + return `${config.host}:${config.port || 1433}/${database} as ${config.user}`; +} diff --git a/src/lib/adapters/database/azure-sql/exporter/index.ts b/src/lib/adapters/database/azure-sql/exporter/index.ts new file mode 100644 index 00000000..c9f99f5b --- /dev/null +++ b/src/lib/adapters/database/azure-sql/exporter/index.ts @@ -0,0 +1,19 @@ +import { sqlpackageExporter } from "./sqlpackage"; +import type { BacpacExporter } from "./types"; + +export type { BacpacExporter, ExporterLog } from "./types"; + +/** + * Which mechanism produces the BACPAC. + * + * One implementation today, and the resolver exists anyway. It is the difference + * between adding the REST mechanism later as a new file plus a branch here, and + * adding it as a change to every call site in dump.ts and restore.ts. + * + * When a second one lands it selects on a config field rather than an environment + * variable: the required credentials differ per source, so one instance has to be + * able to serve both. + */ +export function resolveExporter(): BacpacExporter { + return sqlpackageExporter; +} diff --git a/src/lib/adapters/database/azure-sql/exporter/sqlpackage.ts b/src/lib/adapters/database/azure-sql/exporter/sqlpackage.ts new file mode 100644 index 00000000..75fcc7d9 --- /dev/null +++ b/src/lib/adapters/database/azure-sql/exporter/sqlpackage.ts @@ -0,0 +1,176 @@ +import readline from "node:readline"; +import type { ExecutionHost } from "@/lib/transport"; +import type { AzureSQLConfig } from "@/lib/adapters/definitions"; +import { AdapterError } from "@/lib/logging/errors"; +import { buildConnectionString, describeConnection } from "./connection-string"; +import type { BacpacExporter, ExporterLog } from "./types"; + +/** + * BACPAC export and import through Microsoft's SqlPackage. + * + * ## The connection string is on argv, and that is a deliberate exception + * + * `src/lib/adapters/CLAUDE.md` says secrets go in `options.env`, never in argv. + * SqlPackage has no environment route and rejects `/SourceConnectionString:@file` + * (verified against 170.4.83), so the string has to be an argument. + * + * The exception holds because of why the rule exists: `SshHost` renders env into + * an `export` prefix so secrets stay out of the *remote* process table. This + * adapter has no SSH mode at all, by construction - the schema carries no + * `connectionMode`, so `standardTransport` always returns a DirectHost. The argv + * array therefore never touches a shell, and the exposure is the process table of + * the very container that already holds the password in memory. + * + * That bound is structural, not a `host.kind` check, which the transport lint + * guard forbids for good reason. If an SSH mode is ever added here, this comment + * stops being true and the secret handling has to be revisited first. + */ + +const BINARY = "sqlpackage"; + +/** Lines SqlPackage prefixes with `***` are warnings or errors, never progress. */ +const NOTICE_PREFIX = "***"; + +/** + * Ledger tables cannot be captured completely by a BACPAC. + * + * SqlPackage says so per element, and the consequence is worth raising rather than + * leaving in the noise: the history table and the generated-always columns are + * dropped, which is exactly the tamper evidence Ledger exists to provide. A user + * who reads this in the run log can still decide what to do. One who finds out + * during a restore cannot. + */ +function isLedgerNotice(line: string): boolean { + return line.includes("ledger table") || line.includes("ledger data in system views"); +} + +/** + * Run SqlPackage and stream its output into the run log. + * + * Uses spawn rather than exec because the interesting part is the narration. + * SqlPackage reports each table as it processes it, and buffering that until the + * end would leave a multi-hour export looking hung. + */ +async function run( + argv: string[], + host: ExecutionHost, + log: ExporterLog, + operation: string, + onLine?: (line: string) => void, +): Promise { + const proc = await host.spawn(argv); + + const notices: string[] = []; + let sawLedgerNotice = false; + + const consume = async (stream: NodeJS.ReadableStream, isStderr: boolean) => { + for await (const raw of readline.createInterface({ input: stream, crlfDelay: Infinity })) { + const line = raw.trim(); + if (!line) continue; + + if (line.startsWith(NOTICE_PREFIX)) { + const text = line.slice(NOTICE_PREFIX.length).trim(); + notices.push(text); + + if (isLedgerNotice(text) && !sawLedgerNotice) { + sawLedgerNotice = true; + log( + "This database uses Ledger tables. A BACPAC cannot capture their history tables or their generated-always columns, so the tamper evidence is not part of this backup.", + "warning", + ); + } + log(text, "warning"); + continue; + } + + log(`SqlPackage: ${line}`, isStderr ? "warning" : "info"); + onLine?.(line); + } + }; + + await Promise.all([consume(proc.stdout, false), consume(proc.stderr, true)]); + const { code, signal } = await proc.exit(); + + if (code !== 0) { + // The notices carry the actual cause. The exit code on its own says only + // that something went wrong, which is what made the old MSSQL failures so + // hard to read. + const detail = notices.length > 0 ? notices.join(" | ") : `exit code ${code}${signal ? ` (${signal})` : ""}`; + throw new AdapterError("azure-sql", operation, detail); + } +} + +export const sqlpackageExporter: BacpacExporter = { + id: "sqlpackage", + + async probe(_config: AzureSQLConfig, host: ExecutionHost) { + try { + const binary = await host.which(BINARY); + const result = await host.exec([binary, "/version"], { timeoutMs: 30_000 }); + if (result.code !== 0) { + return { ok: false, detail: `${BINARY} is installed but exited with code ${result.code}` }; + } + return { ok: true, detail: `SqlPackage ${result.stdout.trim()}` }; + } catch { + return { + ok: false, + // Both halves are needed. The container is the normal case, but a + // development checkout has no image at all, and blaming one there + // sends the reader looking for a problem that does not exist. + detail: `${BINARY} was not found on PATH. It ships with the DBackup container image, so in Docker this points at a custom or outdated image. In a local development setup, run scripts/setup-dev-macos.sh or scripts/setup-dev-debian.sh.`, + }; + } + }, + + async exportDatabase(config, dbName, destPath, host, log) { + const binary = await host.which(BINARY); + + log(`Exporting ${describeConnection(config, dbName)}`, "info", "command"); + + await run( + [ + binary, + "/Action:Export", + `/TargetFile:${destPath}`, + "/OverwriteFiles:True", + `/SourceConnectionString:${buildConnectionString(config, dbName)}`, + ], + host, + log, + "export", + ); + }, + + async importDatabase(config, srcPath, targetDbName, host, log, onProgress) { + const binary = await host.which(BINARY); + + log(`Importing into ${describeConnection(config, targetDbName)}`, "info", "command"); + + // SqlPackage reports no percentage of its own, so these are the phases it + // does announce, mapped to anchors. Inventing a smooth curve between them + // would be a guess presented as a measurement. + const anchors: [string, number][] = [ + ["Initializing deployment", 10], + ["Importing package schema", 25], + ["Processing Import", 40], + ["Enabling indexes", 85], + ["Successfully imported", 100], + ]; + + await run( + [ + binary, + "/Action:Import", + `/SourceFile:${srcPath}`, + `/TargetConnectionString:${buildConnectionString(config, targetDbName)}`, + ], + host, + log, + "import", + (line) => { + const anchor = anchors.find(([needle]) => line.includes(needle)); + if (anchor) onProgress?.(anchor[1], line); + }, + ); + }, +}; diff --git a/src/lib/adapters/database/azure-sql/exporter/types.ts b/src/lib/adapters/database/azure-sql/exporter/types.ts new file mode 100644 index 00000000..c3de8dce --- /dev/null +++ b/src/lib/adapters/database/azure-sql/exporter/types.ts @@ -0,0 +1,67 @@ +import type { ExecutionHost } from "@/lib/transport"; +import type { AzureSQLConfig } from "@/lib/adapters/definitions"; +import type { LogLevel, LogType } from "@/lib/core/logs"; + +export type ExporterLog = (msg: string, level?: LogLevel, type?: LogType, details?: string) => void; + +/** + * The single seam between what this adapter orchestrates and how a BACPAC is + * actually produced. + * + * Two mechanisms can fill it. SqlPackage runs a binary in this container and is + * what ships. The Azure Import/Export REST API drives the operation server side + * and needs no binary at all, which mattered while it was unclear whether + * SqlPackage runs on arm64. It does, so the REST path stays unwritten - but the + * seam is what keeps adding it a contained change rather than a rewrite. + * + * Above this interface: database enumeration, multi-database TAR packing, target + * checks, temp file lifecycle. Below it: argv or HTTP construction, credential + * handling, and the wording of the errors. That last one matters more than it + * looks. "sqlpackage: command not found" and "ARM returned 429" are different + * vocabularies, and if the orchestration authored those messages it would grow a + * mechanism switch within two changes. + */ +export interface BacpacExporter { + readonly id: "sqlpackage"; + + /** + * Can this mechanism run here at all? + * + * Deliberately non-throwing, and deliberately called from `test()` rather than + * from `dump()`. A missing binary should surface when someone clicks Test + * Connection, not at 03:00 in a scheduled run. + */ + probe(config: AzureSQLConfig, host: ExecutionHost): Promise<{ ok: boolean; detail: string }>; + + /** + * Export one database to a BACPAC. + * + * `destPath` is a path ON `host`, and the caller is responsible for wrapping it + * in `host.captureOutput`. On a DirectHost that wrapper is a no-op, so the + * discipline costs nothing today and is what keeps a `host.kind` fork from + * growing back if a transport is ever added. + */ + exportDatabase( + config: AzureSQLConfig, + dbName: string, + destPath: string, + host: ExecutionHost, + log: ExporterLog, + ): Promise; + + /** + * Import a BACPAC into a target database that does not exist yet. + * + * `srcPath` is a path ON `host`, wrapped by the caller in `host.stageInput`. + * Unlike the dump path, the runner does pass an `onProgress` through to + * restore, so this one can report percentages. + */ + importDatabase( + config: AzureSQLConfig, + srcPath: string, + targetDbName: string, + host: ExecutionHost, + log: ExporterLog, + onProgress?: (percentage: number, detail?: string) => void, + ): Promise; +} diff --git a/src/lib/adapters/database/azure-sql/identifiers.ts b/src/lib/adapters/database/azure-sql/identifiers.ts new file mode 100644 index 00000000..c6a3a00a --- /dev/null +++ b/src/lib/adapters/database/azure-sql/identifiers.ts @@ -0,0 +1,17 @@ +/** + * Identifier handling, shared with the MSSQL adapter. + * + * Azure SQL Database is the same engine as far as identifiers are concerned: + * `sysname` is still nvarchar(128), bracket quoting still escapes `]` as `]]`, + * and a delimited identifier still accepts hyphens, dots and spaces. Copying the + * rules would mean two places to get them wrong, so they are shared on purpose. + * + * Re-exported through this file rather than imported directly at each call site, + * so the deliberate coupling to the MSSQL adapter is stated once and stays + * visible if either engine ever diverges. + */ +export { + assertValidDatabaseName, + validateDatabaseName, + escapeTSqlString, +} from "../mssql/identifiers"; diff --git a/src/lib/adapters/database/azure-sql/index.ts b/src/lib/adapters/database/azure-sql/index.ts new file mode 100644 index 00000000..574d0e9d --- /dev/null +++ b/src/lib/adapters/database/azure-sql/index.ts @@ -0,0 +1,38 @@ +import { DatabaseAdapter } from "@/lib/core/interfaces"; +import { AzureSQLSchema } from "@/lib/adapters/definitions"; +import { dump } from "./dump"; +import { restore, analyzeDump } from "./restore"; +import { prepareRestore } from "./preflight"; +import { test, getDatabases } from "./connection"; +import { getDatabasesWithStats } from "./catalog"; +import { getTables, getTableData } from "./browser"; + +/** + * Azure SQL Database. + * + * Separate from the MSSQL adapter rather than a mode on it, because almost nothing + * is shared below the wire protocol. Azure SQL Database has no BACKUP DATABASE + * statement, no server-scoped catalog views and no three-part names, so the backup + * format is a BACPAC and every catalog read needs its own connection. + * + * No `transport` resolver and no `connectionMode` in the schema, which is what + * makes `standardTransport` resolve to a DirectHost. That is the correct answer for + * a public PaaS endpoint, and it is also what bounds the argv exception documented + * in exporter/sqlpackage.ts. + */ +export const AzureSQLAdapter: DatabaseAdapter = { + id: "azure-sql", + type: "database", + name: "Azure SQL Database", + configSchema: AzureSQLSchema, + credentials: { primary: "USERNAME_PASSWORD" }, + dump, + restore, + prepareRestore, + test, + getDatabases, + getDatabasesWithStats, + analyzeDump, + getTables, + getTableData, +}; diff --git a/src/lib/adapters/database/azure-sql/pool.ts b/src/lib/adapters/database/azure-sql/pool.ts new file mode 100644 index 00000000..86f50e3c --- /dev/null +++ b/src/lib/adapters/database/azure-sql/pool.ts @@ -0,0 +1,75 @@ +import sql from "mssql"; + +import type { ExecutionHost } from "@/lib/transport"; +import type { AzureSQLConfig } from "@/lib/adapters/definitions"; + +/** + * Opening a TDS connection to Azure SQL Database. + * + * A fork of the MSSQL pool rather than a shared module, for two reasons. A change + * to that one's SSH tunnelling would otherwise silently change this adapter, which + * has no SSH mode. And the settings below are pinned here rather than offered: + * Azure presents a real certificate on every connection, so encryption is not a + * choice and trusting an unverified certificate against *.database.windows.net is + * always either a mistake or an interception. + * + * The `database` option is not a convenience. Azure SQL Database rejects + * three-part names, so every per-database catalog read needs its own connection. + */ + +export interface PoolOptions { + /** Defaults to `master`, which is where the server-scoped catalog lives. */ + database?: string; + /** Override the request timeout, 0 for none. */ + requestTimeout?: number; +} + +function buildConnectionConfig( + config: AzureSQLConfig, + server: string, + port: number, + options: PoolOptions, +): sql.config { + return { + server, + port, + user: config.user, + password: config.password || "", + database: options.database ?? "master", + options: { + encrypt: true, + trustServerCertificate: false, + connectTimeout: 15000, + requestTimeout: options.requestTimeout ?? config.requestTimeout ?? 300000, + }, + }; +} + +/** + * Run `fn` against a connected pool, closing it afterwards. + */ +export async function withPool( + config: AzureSQLConfig, + host: ExecutionHost, + fn: (pool: sql.ConnectionPool) => Promise, + options: PoolOptions = {}, +): Promise { + if (!host) { + throw new Error("Azure SQL adapter requires an execution host. Call it through withHost()."); + } + + // Always a no-op on the DirectHost this adapter resolves to. Kept because it is + // the convention every other adapter follows, and skipping it would be the one + // place a future transport silently bypasses. + const forward = await host.forwardPort(config.host, config.port || 1433); + let pool: sql.ConnectionPool | null = null; + + try { + pool = new sql.ConnectionPool(buildConnectionConfig(config, forward.host, forward.port, options)); + await pool.connect(); + return await fn(pool); + } finally { + if (pool) await pool.close().catch(() => {}); + await forward.close().catch(() => {}); + } +} diff --git a/src/lib/adapters/database/azure-sql/preflight.ts b/src/lib/adapters/database/azure-sql/preflight.ts new file mode 100644 index 00000000..13f6c21a --- /dev/null +++ b/src/lib/adapters/database/azure-sql/preflight.ts @@ -0,0 +1,65 @@ +import type { ExecutionHost } from "@/lib/transport"; +import type { AzureSQLConfig } from "@/lib/adapters/definitions"; +import { logger } from "@/lib/logging/logger"; +import { withPool } from "./pool"; +import { assertValidDatabaseName } from "./identifiers"; + +const log = logger.child({ adapter: "azure-sql" }); + +/** + * Check that a restore can actually land, before an Execution row exists. + * + * An existing target is not an error here. Every other adapter replaces what it + * restores over - MSSQL uses `WITH REPLACE`, PostgreSQL `--clean`, MongoDB + * `--drop` - and the restore dialog already makes the user pick between overwrite + * and rename, showing which databases it would replace. Refusing here would break + * a choice the user has already made deliberately. + * + * A BACPAC import still cannot overwrite in place, so `restore()` drops the target + * first. That is destructive in the same way the other adapters are, with a better + * safety net than most: Azure keeps deleted databases restorable for the retention + * window of its own automated backups. + */ +export async function prepareRestore( + config: AzureSQLConfig, + databases: string[], + host: ExecutionHost, +): Promise { + for (const name of databases) { + assertValidDatabaseName(name); + } + + await withPool(config, host, async (pool) => { + await warnIfCannotCreateDatabases(pool); + }); +} + +/** + * Warn when the login looks unable to create databases. + * + * A warning and not a refusal, deliberately. Creating a database on Azure SQL + * Database needs membership in `dbmanager` in master, but the server administrator + * login holds the same power without being a member, so `IS_ROLEMEMBER` reports 0 + * for an account that will succeed. Refusing on that signal would lock out the most + * common setup of all, which is worse than a restore that fails later with Azure's + * own error. + */ +async function warnIfCannotCreateDatabases(pool: { + request: () => { query: (q: string) => Promise<{ recordset: Record[] }> }; +}): Promise { + try { + const result = await pool.request().query(` + SELECT + IS_ROLEMEMBER('dbmanager') AS is_dbmanager, + IS_MEMBER('db_owner') AS is_db_owner + `); + + const row = result.recordset[0] ?? {}; + if (Number(row.is_dbmanager) !== 1 && Number(row.is_db_owner) !== 1) { + log.warn("Restore target login is not a member of dbmanager", { user: "redacted" }); + } + } catch { + // Not answerable on every tier or for every login. Silence is correct here: + // the question was only ever advisory. + } +} diff --git a/src/lib/adapters/database/azure-sql/restore.ts b/src/lib/adapters/database/azure-sql/restore.ts new file mode 100644 index 00000000..4a40858e --- /dev/null +++ b/src/lib/adapters/database/azure-sql/restore.ts @@ -0,0 +1,218 @@ +import path from "path"; +import type { ExecutionHost } from "@/lib/transport"; +import type { BackupResult } from "@/lib/core/interfaces"; +import type { LogLevel, LogType } from "@/lib/core/logs"; +import type { AzureSQLConfig } from "@/lib/adapters/definitions"; +import { + isMultiDbTar, + readTarManifest, + extractSelectedDatabases, + createTempDir, + cleanupTempDir, + shouldRestoreDatabase, + getTargetDatabaseName, +} from "../common/tar-utils"; +import { resolveExporter } from "./exporter"; +import { withPool } from "./pool"; +import { validateDatabaseName } from "./identifiers"; + +type DatabaseMapping = { originalName: string; targetName: string; selected: boolean }[]; + +/** + * Restore config as it reaches the adapter. + * + * `databaseMapping` arrives as an array despite RestoreInput typing it as a record + * too. The pipeline passes whatever it was given straight through, and every + * adapter reads the array form, so that is what is handled here. + */ +type AzureSQLRestoreConfig = AzureSQLConfig & { + databaseMapping?: DatabaseMapping; + privilegedAuth?: { user: string; password: string }; +}; + +/** One database to import, and where its BACPAC currently sits locally. */ +interface RestoreItem { + localPath: string; + targetName: string; +} + +/** + * Import one or more BACPACs into Azure SQL Database. + * + * Every target is a database that does not exist yet, which prepareRestore has + * already verified. SqlPackage creates it as part of the import. + */ +export async function restore( + config: AzureSQLRestoreConfig, + sourcePath: string, + host: ExecutionHost, + onLog?: (msg: string, level?: LogLevel, type?: LogType, details?: string) => void, + onProgress?: (percentage: number, detail?: string) => void, +): Promise { + const startedAt = new Date(); + const logs: string[] = []; + const log = (msg: string, level: LogLevel = "info", type: LogType = "general", details?: string) => { + logs.push(msg); + onLog?.(msg, level, type, details); + }; + + // The privileged credentials are handed over nested, never flattened, so the + // adapter has to apply them itself. Same pattern as mysql and postgres. + const effectiveConfig: AzureSQLConfig = config.privilegedAuth + ? { ...config, user: config.privilegedAuth.user, password: config.privilegedAuth.password } + : config; + + let stagingDir: string | null = null; + + try { + const exporter = resolveExporter(); + let items: RestoreItem[]; + + if (await isMultiDbTar(sourcePath)) { + stagingDir = await createTempDir("azure-sql-restore-"); + items = await unpackArchive(sourcePath, stagingDir, config.databaseMapping, log); + } else { + items = [{ localPath: sourcePath, targetName: resolveSingleTarget(config) }]; + } + + if (items.length === 0) { + throw new Error("No databases selected for restore."); + } + + for (const item of items) { + log(`Restoring into ${item.targetName}`); + await dropExistingDatabase(effectiveConfig, item.targetName, host, log); + // stageInput is a no-op on a DirectHost, so this hands SqlPackage the + // very file the pipeline already downloaded. + await host.stageInput(item.localPath, {}, (hostPath) => + exporter.importDatabase(effectiveConfig, hostPath, item.targetName, host, log, onProgress), + ); + log(`Restore completed for ${item.targetName}`); + } + + log("Restore finished successfully"); + + return { success: true, path: sourcePath, logs, startedAt, completedAt: new Date() }; + } catch (error: unknown) { + const message = error instanceof Error ? error.message : String(error); + log(`Error: ${message}`, "error"); + return { success: false, logs, error: message, startedAt, completedAt: new Date() }; + } finally { + if (stagingDir) await cleanupTempDir(stagingDir); + } +} + +/** + * Drop the target database so the import can create it. + * + * A BACPAC import always issues its own CREATE DATABASE and has no overwrite mode, + * so replacing a database means dropping it first. That matches what every other + * adapter does on a restore, and the restore dialog already had the user choose + * overwrite over rename before anything got this far. + * + * Destructive, and logged as such. Worth knowing if it was the wrong target: Azure + * keeps a dropped database restorable through **Deleted databases** in the portal + * for the retention window of its own automated backups. + */ +async function dropExistingDatabase( + config: AzureSQLConfig, + name: string, + host: ExecutionHost, + log: (msg: string, level?: LogLevel, type?: LogType, details?: string) => void, +): Promise { + await withPool(config, host, async (pool) => { + const existing = await pool + .request() + .input("dbName", name) + .query("SELECT name FROM sys.databases WHERE name = @dbName"); + + if (existing.recordset.length === 0) return; + + log( + `Dropping the existing database ${name}. A BACPAC import cannot overwrite in place, and Azure can restore a dropped database from Deleted databases in the portal if this was not intended.`, + "warning", + ); + // Bracket quoting through validateDatabaseName, which doubles any `]`. The + // name reaches here from the restore dialog, so it is user input. + await pool.request().query(`DROP DATABASE [${validateDatabaseName(name)}]`); + log(`Dropped ${name}`); + }); +} + +/** + * Extract the selected databases from a multi-database archive. + * + * The manifest is what maps a file inside the archive back to its database name, + * which is why the archive is written with one. Deriving the name from the filename + * instead, as the MSSQL adapter does, breaks on any database whose name contains + * the separator being parsed. + */ +async function unpackArchive( + sourcePath: string, + stagingDir: string, + mapping: DatabaseMapping | undefined, + log: (msg: string, level?: LogLevel) => void, +): Promise { + const manifest = await readTarManifest(sourcePath); + if (!manifest) { + throw new Error("Archive has no manifest.json and cannot be restored."); + } + + const selected = manifest.databases + .map((db) => db.name) + .filter((name) => shouldRestoreDatabase(name, mapping)); + + if (selected.length === 0) return []; + + log(`Extracting ${selected.length} of ${manifest.databases.length} database(s) from the archive`); + const { manifest: extractedManifest, files } = await extractSelectedDatabases(sourcePath, stagingDir, selected); + + // Matched by filename rather than by array position. extractSelectedDatabases + // returns the files it wrote, and nothing promises that order matches the + // manifest once entries have been skipped. + return files.map((localPath) => { + const filename = path.basename(localPath); + const entry = extractedManifest.databases.find((db) => db.filename === filename); + if (!entry) { + throw new Error(`Extracted file ${filename} is not listed in the archive manifest.`); + } + return { localPath, targetName: getTargetDatabaseName(entry.name, mapping) }; + }); +} + +/** Target for a single-database backup, which carries no manifest to consult. */ +function resolveSingleTarget(config: AzureSQLRestoreConfig): string { + const selected = config.databaseMapping?.find((m) => m.selected); + if (selected) return selected.targetName || selected.originalName; + + const database = Array.isArray(config.database) ? config.database[0] : config.database; + if (!database) { + throw new Error("No target database specified for restore."); + } + return database; +} + +/** + * Database names inside a BACPAC. + * + * Unlike a .bak, a BACPAC can be read without a server: it is a ZIP, and a + * multi-database backup is the shared TAR whose manifest lists them outright. + * Implementing this saves the analyze route a full download of the archive purely + * to answer what is inside it. + */ +export async function analyzeDump(sourcePath: string): Promise { + try { + if (await isMultiDbTar(sourcePath)) { + const manifest = await readTarManifest(sourcePath); + return manifest?.databases.map((db) => db.name) ?? []; + } + } catch { + // Unreadable here is not fatal. The caller falls back to the target the user + // picked, and the restore itself will fail with a better message. + } + + // A single BACPAC does not record its source database name in a form worth + // trusting - Origin.xml carries the server it came from, not a name the user + // would recognise as a restore target. + return []; +} diff --git a/src/lib/adapters/database/common/types.ts b/src/lib/adapters/database/common/types.ts index 1fd3f091..7e7dc6ce 100644 --- a/src/lib/adapters/database/common/types.ts +++ b/src/lib/adapters/database/common/types.ts @@ -19,8 +19,8 @@ export interface DatabaseEntry { filename: string; /** Size in bytes (uncompressed) */ size: number; - /** Dump format: sql (MySQL), custom (PostgreSQL -Fc), archive (MongoDB), bak (MSSQL), fbk (Firebird gbak) */ - format: "sql" | "custom" | "archive" | "bak" | "fbk"; + /** Dump format: sql (MySQL), custom (PostgreSQL -Fc), archive (MongoDB), bak (MSSQL), fbk (Firebird gbak), bacpac (Azure SQL Database) */ + format: "sql" | "custom" | "archive" | "bak" | "fbk" | "bacpac"; } /** diff --git a/src/lib/adapters/database/mssql/connection.ts b/src/lib/adapters/database/mssql/connection.ts index 8cbf7f21..55272e27 100644 --- a/src/lib/adapters/database/mssql/connection.ts +++ b/src/lib/adapters/database/mssql/connection.ts @@ -8,8 +8,107 @@ import { MSSQLConfig } from "@/lib/adapters/definitions"; const log = logger.child({ adapter: "mssql" }); /** - * Build connection configuration for mssql package + * `SERVERPROPERTY('EngineEdition')` values. + * + * This is the reliable signal, not the edition string. Azure SQL Database answers + * `SERVERPROPERTY('Edition')` with "SQL Azure", which the name parsing below used + * to reduce to the meaningless "SQL". + */ +const ENGINE_EDITION = { + PERSONAL: 1, + STANDARD: 2, + ENTERPRISE: 3, + EXPRESS: 4, + AZURE_SQL_DATABASE: 5, + AZURE_SYNAPSE: 6, + AZURE_SQL_MANAGED_INSTANCE: 8, + AZURE_SQL_EDGE: 9, + AZURE_SYNAPSE_SERVERLESS: 11, +} as const; + +/** + * Why this adapter structurally cannot back up an engine, or null when it can. + * + * The Azure PaaS editions accept a connection, report a version and list their + * databases, so nothing before the first BACKUP statement gives them away. What + * surfaces there is "Statement 'BACKUP DATABASE' is not supported in this version + * of SQL Server", which names neither the product refusing nor the way forward. + */ +function describeUnsupportedEngine(engineEdition: number): string | null { + switch (engineEdition) { + case ENGINE_EDITION.AZURE_SQL_DATABASE: + return "Azure SQL Database is not supported by this adapter. It has no BACKUP DATABASE statement at all, so a native .bak can never be produced from it."; + case ENGINE_EDITION.AZURE_SQL_MANAGED_INSTANCE: + return "Azure SQL Managed Instance is not supported. It accepts BACKUP DATABASE only as TO URL against Azure Blob Storage with COPY_ONLY, never TO DISK, and this adapter reads the .bak back off a filesystem."; + case ENGINE_EDITION.AZURE_SYNAPSE: + case ENGINE_EDITION.AZURE_SYNAPSE_SERVERLESS: + return "Azure Synapse Analytics is not supported. It has no BACKUP DATABASE statement."; + default: + return null; + } +} + +/** Human-readable edition, keyed off EngineEdition before falling back to the name. */ +function describeEdition(engineEdition: number, editionRaw: string, fullVersion: string): string { + switch (engineEdition) { + case ENGINE_EDITION.AZURE_SQL_DATABASE: return "Azure SQL Database"; + case ENGINE_EDITION.AZURE_SQL_MANAGED_INSTANCE: return "Azure SQL Managed Instance"; + case ENGINE_EDITION.AZURE_SYNAPSE: return "Azure Synapse Analytics"; + case ENGINE_EDITION.AZURE_SYNAPSE_SERVERLESS: return "Azure Synapse Analytics (serverless)"; + case ENGINE_EDITION.AZURE_SQL_EDGE: return "Azure SQL Edge"; + } + + if (fullVersion.includes("Azure SQL Edge")) return "Azure SQL Edge"; + + const lower = editionRaw.toLowerCase(); + if (lower.includes("express")) return "Express"; + if (lower.includes("standard")) return "Standard"; + if (lower.includes("enterprise")) return "Enterprise"; + if (lower.includes("developer")) return "Developer"; + if (lower.includes("web")) return "Web"; + + return editionRaw.split(" ")[0] || "Unknown"; +} + +/** Product name for the connection-test message. */ +function describeProduct(engineEdition: number, fullVersion: string): string { + switch (engineEdition) { + case ENGINE_EDITION.AZURE_SQL_DATABASE: return "Azure SQL Database"; + case ENGINE_EDITION.AZURE_SQL_MANAGED_INSTANCE: return "Azure SQL Managed Instance"; + case ENGINE_EDITION.AZURE_SYNAPSE: + case ENGINE_EDITION.AZURE_SYNAPSE_SERVERLESS: return "Azure Synapse Analytics"; + } + + if (fullVersion.includes("Azure SQL Edge")) return "Azure SQL Edge"; + if (fullVersion.includes("2022")) return "SQL Server 2022"; + if (fullVersion.includes("2019")) return "SQL Server 2019"; + if (fullVersion.includes("2017")) return "SQL Server 2017"; + + return "SQL Server"; +} + +/** + * Refuse an engine this adapter cannot back up, before any work starts. + * + * test() reports the same thing, but a scheduled job never calls test(), and the + * runner swallows its result anyway. Without this the first sign of trouble is a + * failed run at 03:00 quoting a T-SQL error. */ +export async function assertBackupSupported(config: MSSQLConfig, host: ExecutionHost): Promise { + let engineEdition: number; + try { + const result = await executeQuery(config, host, "SELECT SERVERPROPERTY('EngineEdition') AS EngineEdition"); + engineEdition = Number(result.recordset[0]?.EngineEdition) || 0; + } catch { + // A server that will not answer this cannot be classified, and refusing on + // that basis would break setups this adapter has always handled. Let the + // operation continue and fail on its own terms. + return; + } + + const reason = describeUnsupportedEngine(engineEdition); + if (reason) throw new Error(reason); +} /** * Test connection and retrieve version @@ -33,30 +132,17 @@ export async function test(config: MSSQLConfig, host?: ExecutionHost): Promise<{ const versionMatch = productVersion.match(/^(\d+\.\d+\.\d+)/); const version = versionMatch ? versionMatch[1] : productVersion; - // Determine edition string - let edition = "Unknown"; - if (engineEdition === 9 || fullVersion.includes("Azure SQL Edge")) { - edition = "Azure SQL Edge"; - } else if (editionRaw.toLowerCase().includes("express")) { - edition = "Express"; - } else if (editionRaw.toLowerCase().includes("standard")) { - edition = "Standard"; - } else if (editionRaw.toLowerCase().includes("enterprise")) { - edition = "Enterprise"; - } else if (editionRaw.toLowerCase().includes("developer")) { - edition = "Developer"; - } else if (editionRaw.toLowerCase().includes("web")) { - edition = "Web"; - } else { - edition = editionRaw.split(" ")[0] || "Unknown"; // Take first word - } + const edition = describeEdition(engineEdition, editionRaw, fullVersion); + const friendlyName = describeProduct(engineEdition, fullVersion); - // Determine friendly name from full version string - let friendlyName = "SQL Server"; - if (fullVersion.includes("2022")) friendlyName = "SQL Server 2022"; - else if (fullVersion.includes("2019")) friendlyName = "SQL Server 2019"; - else if (fullVersion.includes("2017")) friendlyName = "SQL Server 2017"; - else if (fullVersion.includes("Azure SQL Edge")) friendlyName = "Azure SQL Edge"; + // Reported as a failed test rather than a warning, because a source this + // adapter cannot back up is not a working source. The health check turning + // it offline is what tells the user to switch adapters. Version and edition + // still come back so the run log and version history stay accurate. + const unsupported = describeUnsupportedEngine(engineEdition); + if (unsupported) { + return { success: false, message: unsupported, version, edition }; + } return { success: true, @@ -111,27 +197,14 @@ import { DatabaseInfo } from "@/lib/core/interfaces"; export async function getDatabasesWithStats(config: MSSQLConfig, host: ExecutionHost): Promise { try { return await withPool(config, host, async (pool) => { - // Get database names and sizes from master catalog views. - // Include all user databases regardless of state so offline/restoring DBs - // are still visible. state_desc is included for display purposes. - const sizeResult = await pool.request().query(` - SELECT - d.name, - d.state_desc, - SUM(CAST(mf.size AS BIGINT)) * 8 * 1024 AS size_bytes - FROM sys.databases d - LEFT JOIN sys.master_files mf ON d.database_id = mf.database_id - WHERE d.database_id > 4 - GROUP BY d.name, d.state_desc - ORDER BY d.name - `); + const { rows, sizesAvailable } = await readDatabaseCatalog(pool); // Get table counts per database via cross-database sys.tables queries. // INFORMATION_SCHEMA.TABLES only returns tables for the current DB context, // so we query each database individually. const databases: DatabaseInfo[] = []; - for (const row of sizeResult.recordset) { + for (const row of rows) { let tableCount = 0; try { const safeName = row.name.replace(/\]/g, "]]"); @@ -145,7 +218,10 @@ export async function getDatabasesWithStats(config: MSSQLConfig, host: Execution databases.push({ name: row.name, - sizeInBytes: row.size_bytes != null ? Number(row.size_bytes) : 0, + // Undefined rather than 0 when sizes could not be read at all. The + // explorer drops the whole column when no database reports one, + // which beats a table full of confident zeroes. + sizeInBytes: sizesAvailable ? (row.size_bytes != null ? Number(row.size_bytes) : 0) : undefined, tableCount, }); } @@ -158,6 +234,49 @@ export async function getDatabasesWithStats(config: MSSQLConfig, host: Execution } } +/** + * User databases with their allocated size, falling back to names alone. + * + * `sys.master_files` is server-scoped and does not exist on Azure SQL Database, + * where the join fails with "Invalid object name 'sys.master_files'". Letting that + * escape took out the entire Database Explorer with a "Connection Failed" card, + * even though the connection was fine and the names were perfectly readable. A + * list without sizes beats no list. + * + * All user databases are included regardless of state, so offline and restoring + * ones stay visible. + */ +async function readDatabaseCatalog( + pool: sql.ConnectionPool, +): Promise<{ rows: { name: string; size_bytes?: unknown }[]; sizesAvailable: boolean }> { + try { + const result = await pool.request().query(` + SELECT + d.name, + d.state_desc, + SUM(CAST(mf.size AS BIGINT)) * 8 * 1024 AS size_bytes + FROM sys.databases d + LEFT JOIN sys.master_files mf ON d.database_id = mf.database_id + WHERE d.database_id > 4 + GROUP BY d.name, d.state_desc + ORDER BY d.name + `); + return { rows: result.recordset, sizesAvailable: true }; + } catch (error: unknown) { + log.warn("Database sizes unavailable, listing names only", { + reason: error instanceof Error ? error.message : String(error), + }); + + const result = await pool.request().query(` + SELECT name, state_desc + FROM sys.databases + WHERE database_id > 4 + ORDER BY name + `); + return { rows: result.recordset, sizesAvailable: false }; + } +} + /** * SQL Server message captured during query execution */ diff --git a/src/lib/adapters/database/mssql/dump.ts b/src/lib/adapters/database/mssql/dump.ts index f7a3104a..3a21ceb7 100644 --- a/src/lib/adapters/database/mssql/dump.ts +++ b/src/lib/adapters/database/mssql/dump.ts @@ -1,8 +1,9 @@ import type { ExecutionHost } from "@/lib/transport"; import { BackupResult } from "@/lib/core/interfaces"; import { LogLevel, LogType } from "@/lib/core/logs"; -import { executeQueryWithMessages, getDatabases, supportsCompression, type SqlServerMessage } from "./connection"; +import { assertBackupSupported, executeQueryWithMessages, getDatabases, supportsCompression, type SqlServerMessage } from "./connection"; import { getDialect } from "./dialects"; +import { joinServerPath } from "./server-paths"; import { isCompositeHost } from "@/lib/transport"; import fs from "fs/promises"; import { createReadStream, createWriteStream } from "fs"; @@ -48,6 +49,10 @@ export async function dump( }; try { + // Before anything else, because a scheduled job never runs a connection + // test and the runner discards the one it does run. + await assertBackupSupported(config, host); + // Determine databases to backup let databases: string[] = []; if (Array.isArray(config.database)) { @@ -111,7 +116,7 @@ export async function dump( for (const dbName of databases) { const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const bakFileName = `${dbName}_${timestamp}.bak`; - const serverBakPath = path.posix.join(serverBackupPath, bakFileName); + const serverBakPath = joinServerPath(serverBackupPath, bakFileName); const localBakPath = useSSH ? path.join("/tmp", bakFileName) // SSH mode: always use /tmp locally : path.join(localBackupPath, bakFileName); diff --git a/src/lib/adapters/database/mssql/restore.ts b/src/lib/adapters/database/mssql/restore.ts index a88f471c..68466ea1 100644 --- a/src/lib/adapters/database/mssql/restore.ts +++ b/src/lib/adapters/database/mssql/restore.ts @@ -1,9 +1,10 @@ import type { ExecutionHost } from "@/lib/transport"; import { BackupResult } from "@/lib/core/interfaces"; import { LogLevel, LogType } from "@/lib/core/logs"; -import { executeQuery, executeParameterizedQuery, executeQueryWithMessages, type SqlServerMessage } from "./connection"; +import { assertBackupSupported, executeQuery, executeParameterizedQuery, executeQueryWithMessages, type SqlServerMessage } from "./connection"; import { getDialect } from "./dialects"; import { assertValidDatabaseName, toPhysicalFileName } from "./identifiers"; +import { buildMoveTargets, getInstanceDefaultPaths, joinServerPath, serverDirname, type MoveTarget } from "./server-paths"; import { isCompositeHost } from "@/lib/transport"; import fs from "fs/promises"; import { createReadStream, createWriteStream } from "fs"; @@ -33,6 +34,10 @@ type MSSQLRestoreConfig = MSSQLConfig & { * Prepare restore by validating target databases */ export async function prepareRestore(config: MSSQLRestoreConfig, databases: string[], host: ExecutionHost): Promise { + // Preflight runs before an Execution row exists, so an engine that cannot + // RESTORE at all is rejected here without leaving a failed run behind. + await assertBackupSupported(config, host); + // Check if target databases can be created/overwritten for (const dbName of databases) { // Accept every name SQL Server accepts as a delimited identifier. @@ -140,7 +145,7 @@ export async function restore( const extractedFiles = await extractTarArchive(sourcePath, stagingDir, log, selectedDbNames); for (const extracted of extractedFiles) { - const serverPath = path.posix.join(serverBackupPath, path.basename(extracted)); + const serverPath = joinServerPath(serverBackupPath, path.basename(extracted)); bakFiles.push({ serverPath, localPath: extracted, @@ -151,7 +156,7 @@ export async function restore( } else { // Single .bak file const fileName = path.basename(sourcePath); - const serverBakPath = path.posix.join(serverBackupPath, fileName); + const serverBakPath = joinServerPath(serverBackupPath, fileName); const localBakPath = path.join(stagingDir, fileName); // Stage the file locally first (copy to staging dir) @@ -204,27 +209,25 @@ export async function restore( log(`Restoring database: ${targetDb.original} -> ${targetDb.target}`); - // Build MOVE clauses for file relocation - const moveOptions: { logicalName: string; physicalPath: string }[] = []; - - // The target name becomes a filename here, so path separators have - // to go - same substitution SQL Server applies to its own files. - const fileBaseName = toPhysicalFileName(targetDb.target); - - for (const file of logicalFiles) { - const ext = file.type === "D" ? ".mdf" : ".ldf"; - const newPhysicalPath = `/var/opt/mssql/data/${fileBaseName}${ext}`; - moveOptions.push({ - logicalName: file.logicalName, - physicalPath: newPhysicalPath, - }); + // Only a rename needs MOVE. Restoring under the original name + // leaves the files where the backup already says they belong. + let moveOptions: MoveTarget[] | undefined; + if (targetDb.original !== targetDb.target) { + // The target name becomes a filename here, so path separators have + // to go - same substitution SQL Server applies to its own files. + const fileBaseName = toPhysicalFileName(targetDb.target); + const defaults = await getInstanceDefaultPaths(config, host); + moveOptions = buildMoveTargets(logicalFiles, fileBaseName, defaults); + + const directory = moveOptions.length > 0 ? serverDirname(moveOptions[0].physicalPath) : null; + if (directory) log(`Placing database files in: ${directory}`); } const restoreQuery = dialect.getRestoreQuery(targetDb.target, bakFile.serverPath, { replace: true, recovery: true, stats: 10, - moveFiles: targetDb.original !== targetDb.target ? moveOptions : undefined, + moveFiles: moveOptions, }); log(`Executing restore`, "info", "command", restoreQuery); diff --git a/src/lib/adapters/database/mssql/server-paths.ts b/src/lib/adapters/database/mssql/server-paths.ts new file mode 100644 index 00000000..0da9f071 --- /dev/null +++ b/src/lib/adapters/database/mssql/server-paths.ts @@ -0,0 +1,181 @@ +import type { ExecutionHost } from "@/lib/transport"; +import type { MSSQLConfig } from "@/lib/adapters/definitions"; +import { executeQuery } from "./connection"; +import path from "path"; + +/** + * Paths as SQL Server sees them. + * + * Every other path in this adapter belongs to the machine DBackup runs on, and + * `path` handles those. These belong to the SQL Server host, which is a Windows + * machine often enough that a POSIX assumption is a bug rather than a + * simplification: a Windows server resolves `/var/opt/mssql/data/x.mdf` + * against the current drive and fails with operating system error 3. + */ + +/** ASCII "/" and "\" - compared by code unit so no substring is allocated while scanning. */ +const SLASH = 47; +const BACKSLASH = 92; + +/** Trailing separators of either kind, in one pass. See @/lib/paths for why not a regex. */ +function stripTrailingSeparators(value: string): string { + let end = value.length; + while (end > 0) { + const code = value.charCodeAt(end - 1); + if (code !== SLASH && code !== BACKSLASH) break; + end--; + } + return value.slice(0, end); +} + +/** + * Join a server-side directory and a file name, keeping the separator the + * directory already uses. + * + * Deliberately not "is this a Windows path". A drive letter says nothing about + * the separator: `D:\SQLBackup` and `D:/SQLBackup` both address the same + * directory, but only the second is usable over SFTP, where `\` is an ordinary + * character rather than a separator. Following what the operator wrote keeps + * every existing config producing the exact path it produced before. + * + * `D:\SQLBackup/db.bak` would reach the right file too, since Win32 accepts + * either separator. It is the run log and the error message that make the + * difference: a mangled-looking path is the first thing suspected when + * something else is actually wrong. + */ +export function joinServerPath(directory: string, fileName: string): string { + if (!directory.includes("\\")) { + return path.posix.join(directory, fileName); + } + return `${stripTrailingSeparators(directory)}\\${fileName}`; +} + +/** The directory part of a server-side path, or null when the path carries none. */ +export function serverDirname(value: string): string | null { + const trimmed = stripTrailingSeparators(value); + + let end = trimmed.length; + while (end > 0) { + const code = trimmed.charCodeAt(end - 1); + if (code === SLASH || code === BACKSLASH) break; + end--; + } + if (end === 0) return null; + + // A filesystem root is its own separator, so stripping it leaves nothing. + const directory = stripTrailingSeparators(trimmed.slice(0, end)); + return directory.length > 0 ? directory : trimmed.slice(0, end); +} + +export interface InstanceDefaultPaths { + data?: string; + log?: string; +} + +/** + * The instance's own default data and log directories. + * + * `InstanceDefaultDataPath` exists from SQL Server 2012 SP1 and returns NULL on + * some instances even where it exists, and SERVERPROPERTY answers NULL rather + * than failing for a property an older server has never heard of. Both fields + * are therefore optional and the caller needs a fallback. This never throws: a + * server that will not answer is the same case as one that answers NULL. + */ +export async function getInstanceDefaultPaths( + config: MSSQLConfig, + host: ExecutionHost, +): Promise { + try { + const result = await executeQuery( + config, + host, + "SELECT CAST(SERVERPROPERTY('InstanceDefaultDataPath') AS nvarchar(4000)) AS DataPath, " + + "CAST(SERVERPROPERTY('InstanceDefaultLogPath') AS nvarchar(4000)) AS LogPath", + ); + + const row = result.recordset?.[0] as Record | undefined; + return { data: readPath(row?.DataPath), log: readPath(row?.LogPath) }; + } catch { + return {}; + } +} + +function readPath(value: unknown): string | undefined { + if (typeof value !== "string") return undefined; + const trimmed = value.trim(); + return trimmed.length > 0 ? trimmed : undefined; +} + +/** One row of RESTORE FILELISTONLY, narrowed to the columns that decide placement. */ +export interface RestoreFileEntry { + logicalName: string; + type: string; + physicalName: string; +} + +export interface MoveTarget { + logicalName: string; + physicalPath: string; +} + +/** + * Where the restored files land when the database is being renamed. + * + * Restoring under the original name needs none of this, because the backup + * already records where its files belong. A rename does, since two databases + * cannot share one .mdf. + * + * The instance default comes first: it is where SQL Server would put a new + * database anyway, and it is correct on Windows and Linux alike. The directory + * the file came from is the fallback, which is right whenever the restore + * targets the server that wrote the backup, and that is the normal case for a + * rename. + */ +export function buildMoveTargets( + files: RestoreFileEntry[], + baseName: string, + defaults: InstanceDefaultPaths, +): MoveTarget[] { + let dataFiles = 0; + let logFiles = 0; + + return files.map((file) => { + const type = file.type?.trim().toUpperCase(); + if (type !== "D" && type !== "L") { + // Full-text catalogs and FILESTREAM containers are directories, not + // files, so there is no name to derive here. Restoring under the + // original name still works and is the way to move these. + throw new Error( + `Cannot restore '${file.logicalName}' under a different database name. ` + + `The backup contains a file of type '${file.type}', which SQL Server places by ` + + `directory rather than by file name. Restore under the original database name instead.`, + ); + } + + const isLog = type === "L"; + const directory = (isLog ? defaults.log : defaults.data) ?? serverDirname(file.physicalName); + if (!directory) { + throw new Error( + `Cannot determine where to place '${file.logicalName}' on the server. ` + + `The instance reports no default ${isLog ? "log" : "data"} directory and the backup ` + + `records no directory for this file. Restore under the original database name instead.`, + ); + } + + // A database can hold several data files, and every one of them would + // otherwise be moved onto the same .mdf. + let suffix: string; + if (isLog) { + logFiles++; + suffix = logFiles === 1 ? ".ldf" : `_${logFiles}.ldf`; + } else { + dataFiles++; + suffix = dataFiles === 1 ? ".mdf" : `_${dataFiles}.ndf`; + } + + return { + logicalName: file.logicalName, + physicalPath: joinServerPath(directory, `${baseName}${suffix}`), + }; + }); +} diff --git a/src/lib/adapters/definitions/database.ts b/src/lib/adapters/definitions/database.ts index 74339db1..f03f5911 100644 --- a/src/lib/adapters/definitions/database.ts +++ b/src/lib/adapters/definitions/database.ts @@ -82,6 +82,30 @@ export const MSSQLSchema = z.object({ options: z.string().optional().describe("Additional backup options"), }); +/** + * Azure SQL Database. + * + * Deliberately narrower than MSSQLSchema, and every omission is load bearing: + * + * - No `...sshFields`. The service is a public PaaS endpoint, and the one case an + * SSH tunnel would address, a private endpoint, is not solved by running + * SqlPackage on a jump host that does not have it installed. Leaving + * `connectionMode` out is also what keeps the form on its plain two-tab layout. + * - No `encrypt` or `trustServerCertificate`. Azure presents a real certificate on + * every connection, so both are pinned in `pool.ts` rather than offered. Trusting + * an unverified certificate against *.database.windows.net is always either a + * mistake or an interception. + * - No `backupPath` or `fileTransferMode`. Nothing is ever written server side. + */ +export const AzureSQLSchema = z.object({ + host: z.string().min(1, "Server name is required").describe("Logical server, e.g. myserver.database.windows.net"), + port: z.coerce.number().default(1433), + user: z.string().min(1, "User is required"), + password: z.string().optional(), + database: z.union([z.string(), z.array(z.string())]).default(""), + requestTimeout: z.coerce.number().default(300000).describe("Timeout in ms for catalog queries. The export itself is never timed out."), +}); + export const FirebirdSchema = z.object({ host: z.string().default("localhost"), port: z.coerce.number().default(3050), @@ -120,10 +144,11 @@ export type PostgresConfig = z.infer; export type MongoDBConfig = z.infer; export type SQLiteConfig = z.infer; export type MSSQLConfig = z.infer; +export type AzureSQLConfig = z.infer; export type RedisConfig = z.infer; export type FirebirdConfig = z.infer; -export type DatabaseConfig = MySQLConfig | MariaDBConfig | PostgresConfig | MongoDBConfig | SQLiteConfig | MSSQLConfig | RedisConfig | FirebirdConfig; +export type DatabaseConfig = MySQLConfig | MariaDBConfig | PostgresConfig | MongoDBConfig | SQLiteConfig | MSSQLConfig | AzureSQLConfig | RedisConfig | FirebirdConfig; // Generic type alias for dialect base class (accepts any database config) export type AnyDatabaseConfig = DatabaseConfig; diff --git a/src/lib/adapters/definitions/index.ts b/src/lib/adapters/definitions/index.ts index 97e5f1d1..2ff650ce 100644 --- a/src/lib/adapters/definitions/index.ts +++ b/src/lib/adapters/definitions/index.ts @@ -1,10 +1,10 @@ import { z } from "zod"; import { ADAPTER_CREDENTIAL_REQUIREMENTS } from "@/lib/core/credential-requirements"; -import type { AdapterDefinition } from "./shared"; +import { DEFAULT_S3_UPLOAD_TUNING, type AdapterDefinition } from "./shared"; import { STORAGE_ROLES } from "@/lib/core/storage-roles"; import { MySQLSchema, MariaDBSchema, PostgresSchema, MongoDBSchema, - SQLiteSchema, MSSQLSchema, RedisSchema, FirebirdSchema, + SQLiteSchema, MSSQLSchema, AzureSQLSchema, RedisSchema, FirebirdSchema, } from "./database"; import { LocalStorageSchema, S3GenericSchema, S3AWSSchema, S3R2Schema, S3HetznerSchema, @@ -29,6 +29,7 @@ export const ADAPTER_DEFINITIONS: AdapterDefinition[] = [ { id: "mongodb", type: "database", name: "MongoDB", configSchema: MongoDBSchema }, { id: "sqlite", type: "database", name: "SQLite", configSchema: SQLiteSchema }, { id: "mssql", type: "database", name: "Microsoft SQL Server", configSchema: MSSQLSchema }, + { id: "azure-sql", type: "database", name: "Azure SQL Database", beta: true, configSchema: AzureSQLSchema }, { id: "redis", type: "database", name: "Redis", configSchema: RedisSchema }, { id: "valkey", type: "database", name: "Valkey", configSchema: RedisSchema }, { id: "firebird", type: "database", name: "Firebird", beta: true, configSchema: FirebirdSchema }, @@ -55,10 +56,14 @@ export const ADAPTER_DEFINITIONS: AdapterDefinition[] = [ // restore to one file at a time. transferConcurrency: { default: 4, max: 8 }, }, - { id: "s3-aws", type: "storage", group: "Cloud Storage (S3)", name: "Amazon S3", configSchema: S3AWSSchema }, - { id: "s3-generic", type: "storage", group: "Cloud Storage (S3)", name: "S3 Compatible (Generic)", configSchema: S3GenericSchema }, - { id: "s3-r2", type: "storage", group: "Cloud Storage (S3)", name: "Cloudflare R2", configSchema: S3R2Schema }, - { id: "s3-hetzner", type: "storage", group: "Cloud Storage (S3)", name: "Hetzner Object Storage", configSchema: S3HetznerSchema }, + // The four S3 adapters share one upload path and one range. The AWS SDK's own defaults are + // 4 parts of 5 MB, which measured 27 MB/s against R2 on a 10 Gbit link while everything + // local in the same run moved at over 230 MB/s. See `s3-upload-tuning.ts` for the numbers + // and for why the two values have to be set together. + { id: "s3-aws", type: "storage", group: "Cloud Storage (S3)", name: "Amazon S3", configSchema: S3AWSSchema, multipartUpload: DEFAULT_S3_UPLOAD_TUNING }, + { id: "s3-generic", type: "storage", group: "Cloud Storage (S3)", name: "S3 Compatible (Generic)", configSchema: S3GenericSchema, multipartUpload: DEFAULT_S3_UPLOAD_TUNING }, + { id: "s3-r2", type: "storage", group: "Cloud Storage (S3)", name: "Cloudflare R2", configSchema: S3R2Schema, multipartUpload: DEFAULT_S3_UPLOAD_TUNING }, + { id: "s3-hetzner", type: "storage", group: "Cloud Storage (S3)", name: "Hetzner Object Storage", configSchema: S3HetznerSchema, multipartUpload: DEFAULT_S3_UPLOAD_TUNING }, { id: "google-drive", type: "storage", group: "Cloud Drives", name: "Google Drive", configSchema: GoogleDriveSchema }, { id: "dropbox", type: "storage", group: "Cloud Drives", name: "Dropbox", configSchema: DropboxSchema, @@ -141,6 +146,19 @@ for (const def of ADAPTER_DEFINITIONS) { }); } +// Same reasoning one step narrower: only an adapter that declares multipart upload carries the +// two fields, because on anything else they would be stored and never read. Bounded only by +// what the S3 protocol itself refuses - a part below 5 MB - for the same reason as above, so +// lowering a ceiling in a later version cannot make an existing connection unsaveable. +// Clamping to the adapter's range happens in `resolveS3UploadTuning`. +for (const def of ADAPTER_DEFINITIONS) { + if (!def.multipartUpload) continue; + def.configSchema = def.configSchema.extend({ + uploadConcurrency: z.coerce.number().int().min(1).optional(), + uploadPartSizeMb: z.coerce.number().int().min(5).optional(), + }); +} + export function getAdapterDefinition(id: string) { return ADAPTER_DEFINITIONS.find(d => d.id === id); } diff --git a/src/lib/adapters/definitions/shared.ts b/src/lib/adapters/definitions/shared.ts index c5dc6e9e..00d5c174 100644 --- a/src/lib/adapters/definitions/shared.ts +++ b/src/lib/adapters/definitions/shared.ts @@ -30,6 +30,22 @@ export type AdapterDefinition = { * which is the common case - `DEFAULT_TRANSFER_CONCURRENCY` then applies. */ transferConcurrency?: { default: number; max: number }; + /** + * Storage only: this adapter splits a single upload across parallel parts, and how far the + * connection may push that. + * + * Distinct from `transferConcurrency`, which counts whole files and only applies to a + * directory source. A backup destination receives one archive per run, so the only place + * parallelism can happen is inside that one upload. An adapter that uploads as a single + * stream omits this and shows no field. + * + * Here rather than on the runtime adapter for the same reason as transferConcurrency: the + * connection form runs in the browser, and definitions are plain data. + */ + multipartUpload?: { + concurrency: { default: number; max: number }; + partSizeMb: { default: number; max: number }; + }; /** * Storage only: which roles a config of this adapter may be given. Both, when omitted. * @@ -59,6 +75,22 @@ export type AdapterDefinition = { browseNoun?: string; } +/** + * Applies to an adapter that declares multipart upload without stating its own range. + * + * Eight parts of 8 MB is a deliberate step up from the AWS SDK's own 4 by 5 MB rather than a + * jump to the ceiling. The two numbers multiply into memory, and the default has to stay + * reasonable inside a 512 MB container, which is where most self-hosted installations run. + * + * Lives here rather than next to the resolver in `s3-upload-tuning.ts` because that module + * reads `ADAPTER_DEFINITIONS`, and the definitions need this constant to declare themselves. + * Shared data has to sit below both to keep the import graph acyclic. + */ +export const DEFAULT_S3_UPLOAD_TUNING = { + concurrency: { default: 8, max: 32 }, + partSizeMb: { default: 8, max: 64 }, +} as const; + // Validation: Reject paths with null bytes or obvious shell injection patterns export const safePathRegex = /^[^\0]+$/; export const safePath = (description: string) => diff --git a/src/lib/adapters/index.ts b/src/lib/adapters/index.ts index 5bc8b0ac..3928e6d5 100644 --- a/src/lib/adapters/index.ts +++ b/src/lib/adapters/index.ts @@ -5,6 +5,7 @@ import { PostgresAdapter } from "./database/postgres"; import { MongoDBAdapter } from "./database/mongodb"; import { SQLiteAdapter } from "./database/sqlite"; import { MSSQLAdapter } from "./database/mssql"; +import { AzureSQLAdapter } from "./database/azure-sql"; import { RedisAdapter } from "./database/redis"; import { ValkeyAdapter } from "./database/valkey"; import { FirebirdAdapter } from "./database/firebird"; @@ -46,6 +47,7 @@ export function registerAdapters() { registry.register(MongoDBAdapter); registry.register(SQLiteAdapter); registry.register(MSSQLAdapter); + registry.register(AzureSQLAdapter); registry.register(RedisAdapter); registry.register(ValkeyAdapter); registry.register(FirebirdAdapter); diff --git a/src/lib/adapters/s3-upload-tuning.ts b/src/lib/adapters/s3-upload-tuning.ts new file mode 100644 index 00000000..6a9e72b3 --- /dev/null +++ b/src/lib/adapters/s3-upload-tuning.ts @@ -0,0 +1,162 @@ +/** + * How a single archive is split across parallel connections on the way to an object store. + * + * This is a different question from `transfer-concurrency.ts`, which counts whole *files* and + * only ever applies to a directory source. A backup destination receives one archive per run, + * so there are no files to interleave - the parallelism has to happen inside the one upload, + * across the parts of a multipart request. + * + * Measured against Cloudflare R2 over a 10 Gbit link on 2026-08-09: a 1.39 GB archive took 51s, + * which is 27 MB/s across the four parts the AWS SDK runs by default, so 6.8 MB/s per + * connection. Everything local in the same run moved at 230 to 460 MB/s, and the link was at + * 2% of capacity. The limit is the number of connections, not the line and not the disk. + */ +import type { AdapterConfig } from "@/lib/core/interfaces"; +import { ADAPTER_DEFINITIONS, DEFAULT_S3_UPLOAD_TUNING } from "@/lib/adapters/definitions"; + +export { DEFAULT_S3_UPLOAD_TUNING }; + +const MB = 1024 * 1024; + +/** S3 refuses a part below this, except for the last one. */ +export const S3_MIN_PART_SIZE_MB = 5; + +/** S3 refuses a multipart upload with more parts than this. */ +export const S3_MAX_PARTS = 10_000; + +/** The config keys a connection stores its chosen values under. */ +export const S3_UPLOAD_CONCURRENCY_KEY = "uploadConcurrency"; +export const S3_UPLOAD_PART_SIZE_KEY = "uploadPartSizeMb"; + +export type S3UploadTuningRange = { + concurrency: { default: number; max: number }; + partSizeMb: { default: number; max: number }; +}; + +/** + * The range a given adapter allows, for the form and for clamping. + * + * Looked up by id from the definitions rather than from a runtime adapter, for the same reason + * `transferConcurrencyRange` is: the connection form calls this in the browser, and importing + * the runtime adapters there would pull the AWS SDK into the client bundle. + */ +export function s3UploadTuningRange(adapterId: string): S3UploadTuningRange | undefined { + return ADAPTER_DEFINITIONS.find((d) => d.id === adapterId)?.multipartUpload; +} + +function clampInt(value: unknown, min: number, max: number, fallback: number): number { + const parsed = typeof value === "number" ? value : parseInt(String(value ?? ""), 10); + if (!Number.isFinite(parsed)) return fallback; + return Math.min(max, Math.max(min, Math.floor(parsed))); +} + +export type ResolvedS3UploadTuning = { + /** Parts uploaded at the same time. Maps to the SDK's `queueSize`. */ + queueSize: number; + /** Bytes per part. Maps to the SDK's `partSize`. */ + partSize: number; + /** Why the part size differs from the one the connection asked for, if it does. */ + adjustment: PartSizeAdjustment; +}; + +export type PartSizeAdjustment = + /** The connection's own value was usable as-is. */ + | 'none' + /** The archive would otherwise need more than S3 accepts, so parts had to grow. */ + | 'raised-for-part-limit' + /** The archive would not have produced enough parts to keep every connection busy. */ + | 'lowered-to-fill-parallelism'; + +/** + * How many rounds of work every connection should get. + * + * One would technically keep them all busy, but it makes the upload finish when its *slowest* + * part does, with nothing behind it to absorb a connection that stalls. Two is the cheapest + * number that restores that cushion. + */ +const TARGET_WAVES = 2; + +/** + * Turns a connection's stored values into what `new Upload()` takes. + * + * The stored part size is a ceiling rather than a fixed value, because the number that actually + * performs depends on the archive and the archive differs every run. Both directions are real: + * + * - **Down**, when the archive would not split into enough parts to keep every connection busy. + * Effective parallelism is `min(queueSize, fileSize / partSize)`, so a part size set too high + * silently discards connections. Measured against R2: a 1.39 GB archive at 32 by 64 MB is only + * 21 parts, so 11 connections never received one and the upload took as long as a single part, + * 123 MB/s. The same archive at 32 by 16 MB is 83 parts and ran at 187 MB/s. + * - **Up**, when the archive needs more than the 10.000 parts S3 accepts. Passing an explicit + * `partSize` switches off the SDK's own `max(5 MB, size / 10000)`, and with it the only thing + * keeping a large upload legal. A 500 GB backup at 8 MB parts is 62.500 parts, which S3 + * rejects outright. + * + * The limit wins over the ceiling where they disagree, because the alternative is an upload the + * service refuses. Everything else stays at or below what the connection asked for, so the + * memory figure the form shows is never exceeded. + * + * Leaving `fileSize` undefined skips both, which is correct for a stat that failed: a wrong + * guess about the archive is worse than the configured value. + * + * Stored values are clamped rather than trusted: they arrive from JSON that a restored export + * or a hand-edited database can put anything into, and a ceiling only the form enforces is not + * a ceiling. + */ +export function resolveS3UploadTuning( + adapterId: string, + config: AdapterConfig | undefined, + fileSize?: number +): ResolvedS3UploadTuning { + const range = s3UploadTuningRange(adapterId) ?? DEFAULT_S3_UPLOAD_TUNING; + const stored = config as Record | undefined; + + const queueSize = clampInt( + stored?.[S3_UPLOAD_CONCURRENCY_KEY], + 1, + range.concurrency.max, + range.concurrency.default + ); + const partSizeMb = clampInt( + stored?.[S3_UPLOAD_PART_SIZE_KEY], + S3_MIN_PART_SIZE_MB, + range.partSizeMb.max, + range.partSizeMb.default + ); + + const ceiling = partSizeMb * MB; + if (!fileSize || fileSize <= 0) { + return { queueSize, partSize: ceiling, adjustment: 'none' }; + } + + // Never below the 5 MB S3 refuses: a small archive simply cannot fill every connection, and + // splitting it further would trade a legal upload for parallelism it can never reach. + const fillsEveryConnection = Math.max( + S3_MIN_PART_SIZE_MB * MB, + Math.floor(fileSize / (queueSize * TARGET_WAVES)) + ); + const withinLimit = Math.ceil(fileSize / S3_MAX_PARTS); + + const partSize = Math.max(Math.min(ceiling, fillsEveryConnection), withinLimit); + + return { + queueSize, + partSize, + adjustment: + partSize > ceiling ? 'raised-for-part-limit' + : partSize < ceiling ? 'lowered-to-fill-parallelism' + : 'none', + }; +} + +/** + * Peak bytes held in memory for one upload at these settings. + * + * One part per connection in flight, plus the one the chunker is filling behind them. The form + * shows this because the two fields are meaningless apart: raising parallelism on 64 MB parts + * costs eight times what the same step costs on 8 MB ones, and nobody should have to work that + * out from two spinboxes. + */ +export function s3UploadMemoryBudget(queueSize: number, partSizeMb: number): number { + return (queueSize + 1) * partSizeMb * MB; +} diff --git a/src/lib/adapters/storage/common/read-concurrency.ts b/src/lib/adapters/storage/common/read-concurrency.ts new file mode 100644 index 00000000..b90560ab --- /dev/null +++ b/src/lib/adapters/storage/common/read-concurrency.ts @@ -0,0 +1,18 @@ +/** + * Shared `readConcurrency` value for adapters whose `read()` is stateless. + * + * An adapter opts in by setting `readConcurrency: STATELESS_READ_CONCURRENCY`. That is a + * claim about the protocol, not about the destination: it says a read is a single HTTP + * request or a local file access, with no connection dialled and no process spawned, so + * several can be in flight without anything to exhaust. + * + * Adapters that do open state per read (FTP dials a control connection, SMB spawns an + * smbclient process) deliberately declare nothing and stay serial. There the limit is the + * server's connection count, and going wide turns a slow retention pass into a failing one. + * + * 8 rather than a larger number because the caller is a retention pass reading small + * sidecars. The win is in removing the round trip latency, which is nearly all captured by + * the first handful of parallel requests, and a modest cap keeps a destination holding + * hundreds of backups from opening hundreds of sockets at once. + */ +export const STATELESS_READ_CONCURRENCY = 8; diff --git a/src/lib/adapters/storage/dropbox.ts b/src/lib/adapters/storage/dropbox.ts index d026f128..8d49a6ce 100644 --- a/src/lib/adapters/storage/dropbox.ts +++ b/src/lib/adapters/storage/dropbox.ts @@ -9,6 +9,7 @@ import { LogLevel, LogType } from "@/lib/core/logs"; import { logger } from "@/lib/logging/logger"; import { wrapError } from "@/lib/logging/errors"; import { stripTrailingSlashes } from "@/lib/paths"; +import { STATELESS_READ_CONCURRENCY } from "@/lib/adapters/storage/common/read-concurrency"; const log = logger.child({ adapter: "dropbox" }); @@ -416,6 +417,8 @@ export const DropboxAdapter: StorageAdapter = { } }, + readConcurrency: STATELESS_READ_CONCURRENCY, + async read(config: DropboxConfig, remotePath: string): Promise { try { const dbx = createDropboxClient(config); diff --git a/src/lib/adapters/storage/google-drive.ts b/src/lib/adapters/storage/google-drive.ts index 3d3b0749..93fcea09 100644 --- a/src/lib/adapters/storage/google-drive.ts +++ b/src/lib/adapters/storage/google-drive.ts @@ -9,6 +9,7 @@ import { pipeline } from "stream/promises"; import { LogLevel, LogType } from "@/lib/core/logs"; import { logger } from "@/lib/logging/logger"; import { wrapError } from "@/lib/logging/errors"; +import { STATELESS_READ_CONCURRENCY } from "@/lib/adapters/storage/common/read-concurrency"; const log = logger.child({ adapter: "google-drive" }); @@ -341,6 +342,8 @@ export const GoogleDriveAdapter: StorageAdapter = { } }, + readConcurrency: STATELESS_READ_CONCURRENCY, + async read(config: GoogleDriveConfig, remotePath: string): Promise { try { const drive = createDriveClient(config); diff --git a/src/lib/adapters/storage/local.ts b/src/lib/adapters/storage/local.ts index 866d15c9..bf0c4921 100644 --- a/src/lib/adapters/storage/local.ts +++ b/src/lib/adapters/storage/local.ts @@ -14,6 +14,7 @@ const execFileAsync = promisify(execFile); import { pipeline } from "stream/promises"; import { logger } from "@/lib/logging/logger"; import { wrapError, AdapterError } from "@/lib/logging/errors"; +import { STATELESS_READ_CONCURRENCY } from "@/lib/adapters/storage/common/read-concurrency"; const log = logger.child({ adapter: "local-filesystem" }); @@ -260,6 +261,8 @@ export const LocalFileSystemAdapter: StorageAdapter = { } }, + readConcurrency: STATELESS_READ_CONCURRENCY, + async read(config: { basePath: string }, remotePath: string): Promise { try { const sourcePath = resolveSafePath(config.basePath, remotePath); diff --git a/src/lib/adapters/storage/onedrive.ts b/src/lib/adapters/storage/onedrive.ts index 8e5ee38e..73eeb699 100644 --- a/src/lib/adapters/storage/onedrive.ts +++ b/src/lib/adapters/storage/onedrive.ts @@ -10,6 +10,7 @@ import { LogLevel, LogType } from "@/lib/core/logs"; import { logger } from "@/lib/logging/logger"; import { wrapError } from "@/lib/logging/errors"; import { stripSlashes } from "@/lib/paths"; +import { STATELESS_READ_CONCURRENCY } from "@/lib/adapters/storage/common/read-concurrency"; const log = logger.child({ adapter: "onedrive" }); @@ -421,6 +422,8 @@ export const OneDriveAdapter: StorageAdapter = { } }, + readConcurrency: STATELESS_READ_CONCURRENCY, + async read(config: OneDriveConfig, remotePath: string): Promise { try { const accessToken = await getAccessToken(config); diff --git a/src/lib/adapters/storage/s3.ts b/src/lib/adapters/storage/s3.ts index fa206c09..e23edeed 100644 --- a/src/lib/adapters/storage/s3.ts +++ b/src/lib/adapters/storage/s3.ts @@ -1,8 +1,12 @@ -import { StorageAdapter, FileInfo, DirectoryBrowseEntry, UploadOptions } from "@/lib/core/interfaces"; +import { StorageAdapter, FileInfo, DirectoryBrowseEntry, UploadOptions, ListTreeOptions, ListTreeResult } from "@/lib/core/interfaces"; import { S3GenericSchema, S3AWSSchema, S3R2Schema, S3HetznerSchema } from "@/lib/adapters/definitions"; import { S3Client, ListObjectsV2Command, GetObjectCommand, DeleteObjectCommand, PutObjectCommand, HeadObjectCommand, HeadBucketCommand, StorageClass } from "@aws-sdk/client-s3"; +// Type-only, deliberately. The unit suites replace the whole SDK module with a factory that +// exports the command classes and nothing else, so a value import would break them. +import type { _Object } from "@aws-sdk/client-s3"; import { Upload } from "@aws-sdk/lib-storage"; import { createReadStream, createWriteStream } from "fs"; +import { stat } from "fs/promises"; import { pipeline } from "stream/promises"; import { Transform, Readable } from "stream"; import path from "path"; @@ -10,6 +14,9 @@ import { LogLevel, LogType } from "@/lib/core/logs"; import { logger } from "@/lib/logging/logger"; import { wrapError } from "@/lib/logging/errors"; import { stripSlashes } from "@/lib/paths"; +import { formatBytes } from "@/lib/utils"; +import { resolveS3UploadTuning } from "@/lib/adapters/s3-upload-tuning"; +import { STATELESS_READ_CONCURRENCY } from "@/lib/adapters/storage/common/read-concurrency"; const log = logger.child({ adapter: "s3" }); @@ -23,6 +30,23 @@ interface S3InternalConfig { storageClass?: string; } +/** + * What the upload path needs on top of the connection details. + * + * A separate type rather than three optional fields on `S3InternalConfig`, so the compiler asks + * for `adapterId` at exactly the four call sites that upload and at none of the thirty that + * list, download or delete. Optional everywhere would let one adapter silently miss its wiring + * and fall back to the defaults, which is the one failure this change cannot notice by itself. + */ +interface S3UploadConfig extends S3InternalConfig { + /** Which of the four S3 adapters this is, so the tuning range is looked up correctly. */ + adapterId: string; + /** Parts uploaded at the same time, as stored on the connection. */ + uploadConcurrency?: number; + /** Megabytes per part, as stored on the connection. */ + uploadPartSizeMb?: number; +} + class S3ClientFactory { static create(config: S3InternalConfig) { return new S3Client({ @@ -58,16 +82,44 @@ class S3ClientFactory { // --- Shared Implementation --- -async function s3Upload(internalConfig: S3InternalConfig, localPath: string, remotePath: string, onProgress?: (percent: number) => void, onLog?: (msg: string, level?: LogLevel, type?: LogType, details?: string) => void, options?: UploadOptions): Promise { +async function s3Upload(internalConfig: S3UploadConfig, localPath: string, remotePath: string, onProgress?: (percent: number) => void, onLog?: (msg: string, level?: LogLevel, type?: LogType, details?: string) => void, options?: UploadOptions): Promise { const client = S3ClientFactory.create(internalConfig); const targetKey = S3ClientFactory.getTargetKey(internalConfig, remotePath); if (onLog) onLog(`Starting S3 upload to bucket: ${internalConfig.bucket}, key: ${targetKey}`, 'info', 'storage'); + // The size only picks the part size, so a stat that fails must not fail the upload. Without + // it the configured size is used as-is, which is correct for the small sidecars and the only + // case it could be wrong - an archive past 80 GB needing larger parts to stay under the + // 10.000-part limit - cannot arise from a file the runner has just finished writing. + let fileSize: number | undefined; + try { + fileSize = (await stat(localPath)).size; + } catch { + fileSize = undefined; + } + + const { queueSize, partSize, adjustment } = resolveS3UploadTuning( + internalConfig.adapterId, + internalConfig, + fileSize + ); + + // A file that fits in one part is a plain PutObject however the connection is configured, + // so neither the parallelism nor a transfer rate says anything about it. The metadata + // sidecar is a kilobyte of JSON, and reporting it at "2.88 KB/s" reads like a fault. + const isMultipart = !!fileSize && fileSize > partSize; + const fileStream = createReadStream(localPath); + const startedAt = Date.now(); try { const parallelUploads3 = new Upload({ client: client, + // Both are set explicitly. The SDK's own defaults are 4 parts of 5 MB, which leaves + // most of a fast link idle: measured against R2 over a 10 Gbit line, that is 27 MB/s + // while the same run reads and hashes the file locally at over 400 MB/s. + queueSize, + partSize, params: { Bucket: internalConfig.bucket, Key: targetKey, @@ -77,6 +129,22 @@ async function s3Upload(internalConfig: S3InternalConfig, localPath: string, rem }, }); + // Only worth a line where it says something, and it reports what actually ran rather + // than what the connection stores - the two differ whenever the archive forced a + // different part size, which is exactly when someone reading the log needs to know. + if (onLog && isMultipart) { + const detail = adjustment === 'raised-for-part-limit' + ? ` (raised above the configured maximum to stay within S3's 10,000-part limit)` + : adjustment === 'lowered-to-fill-parallelism' + ? ` (lowered from the configured maximum so every connection gets a part)` + : ''; + onLog( + `Multipart upload: ${queueSize} parallel parts of ${formatBytes(partSize)}${detail}`, + 'info', + 'storage' + ); + } + parallelUploads3.on("httpUploadProgress", (progress) => { if (onProgress && progress.loaded && progress.total) { const percent = Math.round((progress.loaded / progress.total) * 100); @@ -85,7 +153,12 @@ async function s3Upload(internalConfig: S3InternalConfig, localPath: string, rem }); await parallelUploads3.done(); - if (onLog) onLog(`S3 upload completed successfully`, 'info', 'storage'); + // The throughput is the whole reason the two settings above exist, and it used to reach + // only the live progress detail - gone the moment the run ended. Anyone tuning the + // numbers had to rediscover it by subtracting log timestamps, at one-second resolution. + const elapsed = (Date.now() - startedAt) / 1000; + const rate = isMultipart && elapsed > 0 ? ` at ${formatBytes(fileSize! / elapsed)}/s` : ''; + if (onLog) onLog(`S3 upload completed successfully${rate}`, 'info', 'storage'); return true; } catch (error: unknown) { log.error("S3 upload failed", { bucket: internalConfig.bucket, targetKey }, wrapError(error)); @@ -97,33 +170,96 @@ async function s3Upload(internalConfig: S3InternalConfig, localPath: string, rem } } -async function s3List(internalConfig: S3InternalConfig, dir: string = ""): Promise { - const client = S3ClientFactory.create(internalConfig); +/** The prefix a listing scans, with the trailing slash S3 needs to treat it as a folder. */ +function listPrefixFor(internalConfig: S3InternalConfig, dir: string): string { const prefix = S3ClientFactory.getTargetKey(internalConfig, dir); + return prefix && !prefix.endsWith('/') ? `${prefix}/` : prefix; +} - // Ensure prefix ends with / if it serves as a directory listing, unless empty - const listPrefix = prefix && !prefix.endsWith('/') ? `${prefix}/` : prefix; +/** + * Yields every object under a prefix, one ListObjectsV2 page at a time. + * + * ListObjectsV2 answers with at most 1000 keys plus a continuation token, and taking only the + * first page is not a smaller listing - it is the lexicographically first 1000 keys. Backup + * filenames carry timestamps, so that page held the oldest backups and every recent one was + * invisible to retention, integrity checks, the destination browser and the dashboard alike, + * with no error and no log line to say so. + * + * The signal is checked before each request rather than after, so an already-cancelled walk + * costs nothing at all. + * + * No `MaxKeys`, because the 1000 default is what we want. No `Delimiter`, because `list()` is + * deliberately recursive - see the comment in `05-retention.ts` about incremental chains + * living in subfolders. + */ +async function* s3ListPages( + client: S3Client, + bucket: string, + listPrefix: string, + signal?: AbortSignal +): AsyncGenerator<_Object[]> { + let continuationToken: string | undefined; + + do { + signal?.throwIfAborted(); + + const response = await client.send(new ListObjectsV2Command({ + Bucket: bucket, + Prefix: listPrefix, + ContinuationToken: continuationToken, + })); + + yield response.Contents ?? []; + + continuationToken = response.IsTruncated ? response.NextContinuationToken : undefined; + } while (continuationToken); +} + +/** + * Turns one listed object into a `FileInfo`, or `null` for something that is not a file. + * + * Folder markers are recognised by their key ending in `/`, which is what actually makes them + * markers. The old test was `size > 0`, which caught them by accident and threw away every + * genuine empty file with them - so an empty file was missing from a directory backup, read as + * a deleted object during cache reconciliation, and read as a missing link in a backup chain. + * + * The check has to run on the raw key: `path.basename("backups/foo/")` is `"foo"`, so testing + * the name instead would let every marker through. + */ +function s3ObjectToFileInfo(internalConfig: S3InternalConfig, obj: _Object): FileInfo | null { + const key = obj.Key || ""; + if (!key || key.endsWith('/')) return null; + + const name = path.basename(key); + if (!name) return null; + + return { + name, + // Relative to the adapter's path prefix, so it matches every other adapter's + // list() and can be fed straight back to download/delete (which re-apply the + // prefix) without the prefix leaking into stored paths. + path: S3ClientFactory.stripPrefix(internalConfig, key), + size: obj.Size || 0, + lastModified: obj.LastModified || new Date(), + storageClass: obj.StorageClass || undefined, + }; +} + +async function s3List(internalConfig: S3InternalConfig, dir: string = ""): Promise { + const client = S3ClientFactory.create(internalConfig); + const listPrefix = listPrefixFor(internalConfig, dir); try { - const command = new ListObjectsV2Command({ - Bucket: internalConfig.bucket, - Prefix: listPrefix, - }); + const files: FileInfo[] = []; - const response = await client.send(command); + for await (const page of s3ListPages(client, internalConfig.bucket, listPrefix)) { + for (const obj of page) { + const file = s3ObjectToFileInfo(internalConfig, obj); + if (file) files.push(file); + } + } - if (!response.Contents) return []; - - return response.Contents.map(obj => ({ - name: path.basename(obj.Key || ""), - // Relative to the adapter's path prefix, so it matches every other adapter's - // list() and can be fed straight back to download/delete (which re-apply the - // prefix) without the prefix leaking into stored paths. - path: S3ClientFactory.stripPrefix(internalConfig, obj.Key || ""), - size: obj.Size || 0, - lastModified: obj.LastModified || new Date(), - storageClass: obj.StorageClass || undefined, - })).filter(f => f.name && f.size > 0); // Filter folders or empty keys + return files; } catch (error) { log.error("S3 list failed", { bucket: internalConfig.bucket, prefix: listPrefix }, wrapError(error)); throw error; @@ -132,6 +268,61 @@ async function s3List(internalConfig: S3InternalConfig, dir: string = ""): Promi } } +/** + * Collection walk over a prefix, which is `list()` plus progress and cancellation. + * + * Same pagination helper as `list()`, on purpose. Two loops would drift, and retention and + * collection would end up disagreeing about what is in a destination - the reason `ftp.ts` + * keeps one walker behind both of its entry points. + * + * Without this, `listTreeForCollection()` falls back to `list()`, which reports progress only + * once it has finished and cannot be interrupted at all. That was harmless while a listing + * stopped at 1000 keys and is not once it paginates: a large bucket lists for minutes behind a + * frozen progress row and a cancel button that does nothing. + * + * `pruned` is always empty, and that is a decision rather than an omission. `excludePatterns` + * are advisory and the caller applies them again anyway, a flat scan has no directory it could + * decline to descend into, and rebuilding this as a `Delimiter` walk to gain one would cost S3 + * more requests rather than fewer. `concurrency` is ignored for the same kind of reason: + * pagination is serial by construction, because the next token only exists once the previous + * response has arrived. `unsupportedSymlinks` stays unset - object storage has no links. + */ +async function s3ListTree( + internalConfig: S3InternalConfig, + dir: string = "", + options?: ListTreeOptions +): Promise { + const client = S3ClientFactory.create(internalConfig); + const listPrefix = listPrefixFor(internalConfig, dir); + + try { + const files: FileInfo[] = []; + + for await (const page of s3ListPages(client, internalConfig.bucket, listPrefix, options?.signal)) { + for (const obj of page) { + const file = s3ObjectToFileInfo(internalConfig, obj); + if (file) files.push(file); + } + + // One report per page. No self-throttling: listTreeForCollection() already rate + // limits what reaches the execution's progress row. + options?.onProgress?.({ + files: files.length, + directories: 0, + prunedDirectories: 0, + currentPath: "", + }); + } + + return { files, pruned: [] }; + } catch (error) { + log.error("S3 tree listing failed", { bucket: internalConfig.bucket, prefix: listPrefix }, wrapError(error)); + throw error; + } finally { + client.destroy(); + } +} + /** * Lists one level of "folders" below a prefix. * @@ -145,8 +336,7 @@ async function s3BrowseDirectories( subPath: string = "" ): Promise { const client = S3ClientFactory.create(internalConfig); - const base = S3ClientFactory.getTargetKey(internalConfig, subPath); - const listPrefix = base && !base.endsWith("/") ? `${base}/` : base; + const listPrefix = listPrefixFor(internalConfig, subPath); try { const entries: DirectoryBrowseEntry[] = []; @@ -405,12 +595,15 @@ export const S3GenericAdapter: StorageAdapter = { configSchema: S3GenericSchema, credentials: { primary: "ACCESS_KEY" }, upload: (config, ...args) => s3Upload({ + adapterId: "s3-generic", endpoint: config.endpoint, region: config.region, bucket: config.bucket, credentials: { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey }, forcePathStyle: config.forcePathStyle, - pathPrefix: config.pathPrefix + pathPrefix: config.pathPrefix, + uploadConcurrency: config.uploadConcurrency, + uploadPartSizeMb: config.uploadPartSizeMb }, ...args), list: (config, ...args) => s3List({ endpoint: config.endpoint, @@ -420,6 +613,14 @@ export const S3GenericAdapter: StorageAdapter = { forcePathStyle: config.forcePathStyle, pathPrefix: config.pathPrefix }, ...args), + listTree: (config, ...args) => s3ListTree({ + endpoint: config.endpoint, + region: config.region, + bucket: config.bucket, + credentials: { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey }, + forcePathStyle: config.forcePathStyle, + pathPrefix: config.pathPrefix + }, ...args), download: (config, ...args) => s3Download({ endpoint: config.endpoint, region: config.region, @@ -466,6 +667,7 @@ export const S3GenericAdapter: StorageAdapter = { credentials: { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey }, forcePathStyle: config.forcePathStyle, }), + readConcurrency: STATELESS_READ_CONCURRENCY, read: (config, ...args) => s3Read({ endpoint: config.endpoint, region: config.region, @@ -492,11 +694,14 @@ export const S3AWSAdapter: StorageAdapter = { configSchema: S3AWSSchema, credentials: { primary: "ACCESS_KEY" }, upload: (config, ...args) => s3Upload({ + adapterId: "s3-aws", region: config.region, bucket: config.bucket, credentials: { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey }, pathPrefix: config.pathPrefix, - storageClass: config.storageClass + storageClass: config.storageClass, + uploadConcurrency: config.uploadConcurrency, + uploadPartSizeMb: config.uploadPartSizeMb }, ...args), list: (config, ...args) => s3List({ region: config.region, @@ -504,6 +709,12 @@ export const S3AWSAdapter: StorageAdapter = { credentials: { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey }, pathPrefix: config.pathPrefix }, ...args), + listTree: (config, ...args) => s3ListTree({ + region: config.region, + bucket: config.bucket, + credentials: { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey }, + pathPrefix: config.pathPrefix + }, ...args), download: (config, ...args) => s3Download({ region: config.region, bucket: config.bucket, @@ -538,6 +749,7 @@ export const S3AWSAdapter: StorageAdapter = { bucket: config.bucket, credentials: { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey }, }), + readConcurrency: STATELESS_READ_CONCURRENCY, read: (config, ...args) => s3Read({ region: config.region, bucket: config.bucket, @@ -566,11 +778,14 @@ export const S3R2Adapter: StorageAdapter = { configSchema: S3R2Schema, credentials: { primary: "ACCESS_KEY" }, upload: (config, ...args) => s3Upload({ + adapterId: "s3-r2", endpoint: r2Endpoint(config.accountId, config.jurisdiction), region: "auto", bucket: config.bucket, credentials: { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey }, - pathPrefix: config.pathPrefix + pathPrefix: config.pathPrefix, + uploadConcurrency: config.uploadConcurrency, + uploadPartSizeMb: config.uploadPartSizeMb }, ...args), list: (config, ...args) => s3List({ endpoint: r2Endpoint(config.accountId, config.jurisdiction), @@ -579,6 +794,13 @@ export const S3R2Adapter: StorageAdapter = { credentials: { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey }, pathPrefix: config.pathPrefix }, ...args), + listTree: (config, ...args) => s3ListTree({ + endpoint: r2Endpoint(config.accountId, config.jurisdiction), + region: "auto", + bucket: config.bucket, + credentials: { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey }, + pathPrefix: config.pathPrefix + }, ...args), download: (config, ...args) => s3Download({ endpoint: r2Endpoint(config.accountId, config.jurisdiction), region: "auto", @@ -619,6 +841,7 @@ export const S3R2Adapter: StorageAdapter = { bucket: config.bucket, credentials: { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey }, }), + readConcurrency: STATELESS_READ_CONCURRENCY, read: (config, ...args) => s3Read({ endpoint: r2Endpoint(config.accountId, config.jurisdiction), region: "auto", @@ -643,11 +866,14 @@ export const S3HetznerAdapter: StorageAdapter = { configSchema: S3HetznerSchema, credentials: { primary: "ACCESS_KEY" }, upload: (config, ...args) => s3Upload({ + adapterId: "s3-hetzner", endpoint: `https://${config.region}.your-objectstorage.com`, region: config.region, bucket: config.bucket, credentials: { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey }, - pathPrefix: config.pathPrefix + pathPrefix: config.pathPrefix, + uploadConcurrency: config.uploadConcurrency, + uploadPartSizeMb: config.uploadPartSizeMb }, ...args), list: (config, ...args) => s3List({ endpoint: `https://${config.region}.your-objectstorage.com`, @@ -656,6 +882,13 @@ export const S3HetznerAdapter: StorageAdapter = { credentials: { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey }, pathPrefix: config.pathPrefix }, ...args), + listTree: (config, ...args) => s3ListTree({ + endpoint: `https://${config.region}.your-objectstorage.com`, + region: config.region, + bucket: config.bucket, + credentials: { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey }, + pathPrefix: config.pathPrefix + }, ...args), download: (config, ...args) => s3Download({ endpoint: `https://${config.region}.your-objectstorage.com`, region: config.region, @@ -696,6 +929,7 @@ export const S3HetznerAdapter: StorageAdapter = { bucket: config.bucket, credentials: { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey }, }), + readConcurrency: STATELESS_READ_CONCURRENCY, read: (config, ...args) => s3Read({ endpoint: `https://${config.region}.your-objectstorage.com`, region: config.region, diff --git a/src/lib/adapters/storage/webdav.ts b/src/lib/adapters/storage/webdav.ts index e5ed4f60..fc8cd100 100644 --- a/src/lib/adapters/storage/webdav.ts +++ b/src/lib/adapters/storage/webdav.ts @@ -9,6 +9,7 @@ import { pipeline } from "stream/promises"; import { LogLevel, LogType } from "@/lib/core/logs"; import { logger } from "@/lib/logging/logger"; import { wrapError } from "@/lib/logging/errors"; +import { STATELESS_READ_CONCURRENCY } from "@/lib/adapters/storage/common/read-concurrency"; const log = logger.child({ adapter: "webdav" }); @@ -138,6 +139,8 @@ export const WebDAVAdapter: StorageAdapter = { } }, + readConcurrency: STATELESS_READ_CONCURRENCY, + async read(config: WebDAVConfig, remotePath: string): Promise { try { const client = getClient(config); diff --git a/src/lib/auth/adapter-permissions.ts b/src/lib/auth/adapter-permissions.ts index a65340e5..30d4d98e 100644 --- a/src/lib/auth/adapter-permissions.ts +++ b/src/lib/auth/adapter-permissions.ts @@ -7,6 +7,7 @@ const ADAPTER_PERMISSIONS: Record = { mongodb: PERMISSIONS.SOURCES.VIEW, sqlite: PERMISSIONS.SOURCES.VIEW, mssql: PERMISSIONS.SOURCES.VIEW, + "azure-sql": PERMISSIONS.SOURCES.VIEW, redis: PERMISSIONS.SOURCES.VIEW, valkey: PERMISSIONS.SOURCES.VIEW, firebird: PERMISSIONS.SOURCES.VIEW, diff --git a/src/lib/backup-extensions.ts b/src/lib/backup-extensions.ts index 4457aa98..2a3deda6 100644 --- a/src/lib/backup-extensions.ts +++ b/src/lib/backup-extensions.ts @@ -19,6 +19,7 @@ export function getBackupFileExtension(adapterId: string): string { mariadb: "sql", postgres: "sql", mssql: "bak", + "azure-sql": "bacpac", // SqlPackage data-tier application export // NoSQL and special formats mongodb: "archive", // mongodump --archive format @@ -45,6 +46,7 @@ export function getBackupFormatDescription(adapterId: string): string { mariadb: "MariaDB SQL Dump", postgres: "PostgreSQL SQL Dump", mssql: "SQL Server Native Backup", + "azure-sql": "Azure SQL Database BACPAC", mongodb: "MongoDB Archive", redis: "Redis RDB Snapshot", valkey: "Valkey RDB Snapshot", diff --git a/src/lib/core/backup-files.ts b/src/lib/core/backup-files.ts index c7c4a537..a2ff8c4e 100644 --- a/src/lib/core/backup-files.ts +++ b/src/lib/core/backup-files.ts @@ -14,6 +14,7 @@ */ import { INDEX_SIDECAR_SUFFIX } from "@/lib/archive/format"; +import type { FileInfo } from "@/lib/core/interfaces"; /** Metadata sidecar written for every backup. */ export const METADATA_SIDECAR_SUFFIX = ".meta.json"; @@ -41,6 +42,22 @@ export function isBackupFile(name: string): boolean { return !isSidecarFile(name); } +/** + * The time a backup should be judged by. + * + * `lastModified` is whatever the destination reports as the file's modification time, and + * that is not the same thing as when the backup was taken. A copy without `-p`, a move + * between servers, or restoring the backup directory itself resets every mtime to now, at + * which point retention sees one giant bucket and deletes almost everything in it. + * + * `backupTimestamp` comes from the backup's own sidecar, written by DBackup at upload, and + * survives all of that. It is missing for backups that predate it and for destinations + * whose adapter cannot read, so the mtime remains the fallback rather than the rule. + */ +export function effectiveBackupTime(file: FileInfo): Date { + return file.backupTimestamp ?? file.lastModified; +} + /** * Every sidecar path belonging to a backup. * diff --git a/src/lib/core/credential-requirements.ts b/src/lib/core/credential-requirements.ts index 80d25973..2ba641ce 100644 --- a/src/lib/core/credential-requirements.ts +++ b/src/lib/core/credential-requirements.ts @@ -21,6 +21,9 @@ export const ADAPTER_CREDENTIAL_REQUIREMENTS: Record< postgres: { primary: "USERNAME_PASSWORD", ssh: "SSH_KEY" }, mongodb: { primary: "USERNAME_PASSWORD", ssh: "SSH_KEY" }, mssql: { primary: "USERNAME_PASSWORD", ssh: "SSH_KEY" }, + // No SSH slot: Azure SQL Database is a public PaaS endpoint, and a tunnel would + // not help anyway since SqlPackage runs in this container, not on a jump host. + "azure-sql": { primary: "USERNAME_PASSWORD" }, redis: { primary: "USERNAME_PASSWORD", ssh: "SSH_KEY" }, valkey: { primary: "USERNAME_PASSWORD", ssh: "SSH_KEY" }, firebird: { primary: "USERNAME_PASSWORD", ssh: "SSH_KEY" }, diff --git a/src/lib/core/interfaces.ts b/src/lib/core/interfaces.ts index 9d13d8b2..8d7f8f0d 100644 --- a/src/lib/core/interfaces.ts +++ b/src/lib/core/interfaces.ts @@ -352,6 +352,16 @@ export type FileInfo = { path: string; size: number; lastModified: Date; + /** + * Creation time DBackup itself recorded in the backup's `.meta.json`, when one was + * readable. + * + * This is what retention buckets by, because it survives the things that reset a + * filesystem mtime: a copy without -p, a migration between servers, a restored backup + * directory. `lastModified` keeps meaning what it says, the mtime the destination + * reports, so the two can be compared when they disagree. + */ + backupTimestamp?: Date; /** * Set when this entry is a symbolic link, holding its raw target exactly as stored on the * source - relative stays relative, and nothing is resolved. @@ -681,6 +691,17 @@ export interface StorageAdapter extends BaseAdapter { */ read?(config: AdapterConfig, remotePath: string): Promise; + /** + * How many `read()` calls a caller may run against this destination at once. + * + * Defaults to serial (1) when unset, which is what every adapter did before this + * existed. Only declare more where `read()` opens no protocol state per call, so an + * HTTP request or a local file access. An adapter that dials a connection or spawns a + * process per read stays serial: there the server's connection limit is what breaks, + * not the bandwidth. + */ + readConcurrency?: number; + /** * Optional: streams a byte range [start, end] (both inclusive) of a remote file. * diff --git a/src/lib/core/retention.ts b/src/lib/core/retention.ts index 0a1288bc..88e7e7b4 100644 --- a/src/lib/core/retention.ts +++ b/src/lib/core/retention.ts @@ -1,3 +1,5 @@ +import { z } from "zod"; + export type RetentionMode = "NONE" | "SIMPLE" | "SMART"; export interface SimpleRetentionPolicy { @@ -5,6 +7,14 @@ export interface SimpleRetentionPolicy { } export interface SmartRetentionPolicy { + /** + * Keep one per hour for X hours. + * + * Optional because every policy written before this tier existed has no value for it, + * and an absent tier has to behave exactly like a disabled one. Treat a missing value + * as 0 rather than passing it through to a comparison. + */ + hourly?: number; daily: number; // Keep one per day for X days weekly: number; // Keep one per week for X weeks monthly: number; // Keep one per month for X months @@ -20,3 +30,34 @@ export interface RetentionConfiguration { export const DEFAULT_RETENTION_CONFIG: RetentionConfiguration = { mode: "NONE", }; + +/** Value pre-filled when the hourly tier is switched on in the policy form. */ +export const DEFAULT_HOURLY_TIER = 24; + +/** A tier limit. 0 disables the tier, fractions and negatives are rejected. */ +const tierLimit = z.coerce.number().int().min(0); + +export const SimpleRetentionPolicySchema = z.object({ + keepCount: z.coerce.number().int().min(1), +}); + +export const SmartRetentionPolicySchema = z.object({ + hourly: tierLimit.optional(), + daily: tierLimit, + weekly: tierLimit, + monthly: tierLimit, + yearly: tierLimit, +}); + +/** + * Validates a retention configuration before it is stored. + * + * Retention deletes files, so a config that reaches the engine has to be sound. Without + * this a value written through the API, such as a negative or non numeric tier, lands in + * the bucketing logic unchecked. + */ +export const RetentionConfigurationSchema = z.object({ + mode: z.enum(["NONE", "SIMPLE", "SMART"]), + simple: SimpleRetentionPolicySchema.optional(), + smart: SmartRetentionPolicySchema.optional(), +}); diff --git a/src/lib/incompressible-formats.ts b/src/lib/incompressible-formats.ts index eeb8079e..84bdc792 100644 --- a/src/lib/incompressible-formats.ts +++ b/src/lib/incompressible-formats.ts @@ -33,8 +33,8 @@ export const INCOMPRESSIBLE_EXTENSIONS: ReadonlySet = new Set([ "7z", "br", "bz2", "cab", "gz", "lz4", "lzma", "rar", "tbz2", "tgz", "txz", "xz", "zip", "zst", // ZIP containers under another name. Office documents, Java and mobile packages, Python - // wheels, browser and editor extensions - all deflate inside. - "apk", "docx", "epub", "ipa", "jar", "nupkg", "odp", "ods", "odt", "pptx", "vsix", "war", "whl", "xlsx", "xpi", + // wheels, browser and editor extensions, Azure SQL data-tier exports - all deflate inside. + "apk", "bacpac", "docx", "epub", "ipa", "jar", "nupkg", "odp", "ods", "odt", "pptx", "vsix", "war", "whl", "xlsx", "xpi", // Web fonts. WOFF and WOFF2 are the compressed forms of TTF and OTF. "woff", "woff2", diff --git a/src/lib/runner/steps/05-retention.ts b/src/lib/runner/steps/05-retention.ts index 5870f136..8eff6adb 100644 --- a/src/lib/runner/steps/05-retention.ts +++ b/src/lib/runner/steps/05-retention.ts @@ -1,7 +1,8 @@ import { RunnerContext, DestinationContext } from "../types"; import { RetentionService } from "@/services/backup/retention-service"; import { FileInfo } from '@/lib/core/interfaces'; -import { isBackupFile, sidecarPathsFor } from '@/lib/core/backup-files'; +import { isBackupFile, sidecarPathsFor, effectiveBackupTime } from '@/lib/core/backup-files'; +import { loadBackupSidecars } from './retention-sidecars'; import path from "path"; import { logger } from "@/lib/logging/logger"; import prisma from "@/lib/prisma"; @@ -90,31 +91,37 @@ async function applyRetentionForDestination(ctx: RunnerContext, dest: Destinatio const files: FileInfo[] = await dest.adapter.list(dest.config, remoteDir); const backupFiles = files.filter(f => isBackupFile(f.name)); - // Read each backup's sidecar for its lock flag and chain membership. The chain id is - // what lets retention treat an incremental chain as one indivisible unit. - if (dest.adapter.read) { - for (const file of backupFiles) { - try { - const metaContent = await dest.adapter.read(dest.config, file.path + ".meta.json"); - if (metaContent) { - const meta = JSON.parse(metaContent); - if (meta.locked) { - file.locked = true; - } - if (meta.chain?.id) { - file.chainId = meta.chain.id; - } - } - } catch (_e) { - // Ignore read errors - } - } + // Each backup's sidecar carries its lock flag, its chain membership and the time + // DBackup recorded when it wrote the backup. The chain id is what lets retention treat + // an incremental chain as one indivisible unit, and the timestamp is what it buckets by. + const sidecars = await loadBackupSidecars(dest.adapter, dest.config, files, backupFiles); + + if (backupFiles.length > 0) { + ctx.log( + `${destLabel} Retention: ${sidecars.withTimestamp} of ${backupFiles.length} backup(s) supplied their own creation time, the rest fall back to the file's modification time on the destination.` + ); + } + // A destination whose modification times were reset, by a copy without -p or by a + // restore of the backup directory, would otherwise collapse into one bucket without + // anyone noticing until backups were already gone. + for (const { file, recorded, modified } of sidecars.drifted) { + ctx.log( + `${destLabel} Retention: ${file.name} was written ${recorded.toISOString()} but the destination reports ${modified.toISOString()}. Retention uses the recorded time.`, + 'warning' + ); } - // Log each file with its timestamp so adapter-level timestamp issues are immediately visible. - const sorted = [...backupFiles].sort((a, b) => b.lastModified.getTime() - a.lastModified.getTime()); + // Log each file with the time it is actually judged by, so a bucketing surprise can be + // traced without guessing which of the two times was used. + const sorted = [...backupFiles].sort( + (a, b) => effectiveBackupTime(b).getTime() - effectiveBackupTime(a).getTime() + ); for (const f of sorted) { - ctx.log(`${destLabel} Retention: Found file: ${f.name} (${f.lastModified.toISOString()})`); + const effective = effectiveBackupTime(f); + const mtimeNote = effective.getTime() === f.lastModified.getTime() + ? '' + : ` (mtime ${f.lastModified.toISOString()})`; + ctx.log(`${destLabel} Retention: Found file: ${f.name} (${effective.toISOString()})${mtimeNote}`); } const { keep, delete: filesToDelete, keptForChain } = RetentionService.calculateRetention(backupFiles, policy, timezone); diff --git a/src/lib/runner/steps/retention-sidecars.ts b/src/lib/runner/steps/retention-sidecars.ts new file mode 100644 index 00000000..fdef23a0 --- /dev/null +++ b/src/lib/runner/steps/retention-sidecars.ts @@ -0,0 +1,109 @@ +import { FileInfo, StorageAdapter, AdapterConfig } from "@/lib/core/interfaces"; +import { METADATA_SIDECAR_SUFFIX } from "@/lib/core/backup-files"; + +/** + * Loading the `.meta.json` sidecars that retention needs before it can decide anything. + * + * Three things live in there that a storage listing cannot tell us: whether a backup is + * locked, which incremental chain it belongs to, and when DBackup actually wrote it. The + * last one matters most, because the destination's own mtime is not trustworthy - see + * `backupTimestamp` on FileInfo. + * + * This runs at the end of every successful job, once per destination, over every backup + * present. That is the reason it is worth being careful about how many round trips it + * costs rather than just looping. + */ + +/** How far mtime and the recorded creation time may drift before it is worth reporting. */ +export const TIMESTAMP_DRIFT_WARNING_MS = 60 * 60 * 1000; + +export interface SidecarLoadResult { + /** Backups whose sidecar supplied a usable creation time. */ + withTimestamp: number; + /** + * Backups whose mtime disagrees with the recorded creation time by more than the + * threshold. Reported by name, because a destination whose modification times were + * reset is otherwise invisible until it has already cost backups. + */ + drifted: { file: FileInfo; recorded: Date; modified: Date }[]; +} + +const normalize = (p: string) => p.replace(/\\/g, "/"); + +/** + * Reads each backup's sidecar and annotates the FileInfo objects in place. + * + * @param listing The complete `list()` output, sidecars included. Used to skip reads for + * backups that have no sidecar at all, which costs nothing to check and + * saves a full round trip per pre-sidecar backup. + * @param backups The subset the caller intends to apply retention to. + */ +export async function loadBackupSidecars( + adapter: StorageAdapter, + config: AdapterConfig, + listing: FileInfo[], + backups: FileInfo[] +): Promise { + const result: SidecarLoadResult = { withTimestamp: 0, drifted: [] }; + if (!adapter.read) return result; + + const sidecarPaths = new Set( + listing + .filter((f) => f.name.endsWith(METADATA_SIDECAR_SUFFIX)) + .map((f) => normalize(f.path)) + ); + + // An adapter whose list() filters sidecars out would otherwise lose lock and chain + // detection entirely, which is far worse than a wasted round trip. Only trust the + // listing to answer "is there a sidecar" when it demonstrably reports them. + const listingShowsSidecars = sidecarPaths.size > 0; + + const targets = listingShowsSidecars + ? backups.filter((f) => sidecarPaths.has(normalize(f.path) + METADATA_SIDECAR_SUFFIX)) + : backups; + + // Serial unless the adapter says its read() carries no per-call protocol state. + const concurrency = Math.max(1, adapter.readConcurrency ?? 1); + + for (let i = 0; i < targets.length; i += concurrency) { + const batch = targets.slice(i, i + concurrency); + await Promise.all(batch.map((file) => applySidecar(adapter, config, file, result))); + } + + return result; +} + +async function applySidecar( + adapter: StorageAdapter, + config: AdapterConfig, + file: FileInfo, + result: SidecarLoadResult +): Promise { + let meta: { locked?: boolean; chain?: { id?: string }; timestamp?: string }; + try { + const content = await adapter.read!(config, file.path + METADATA_SIDECAR_SUFFIX); + if (!content) return; + meta = JSON.parse(content); + } catch { + // A backup whose sidecar cannot be read is still a backup. It falls back to the + // destination's mtime and counts as unlocked and chainless, which is what the + // policy assumed before sidecars existed. + return; + } + + if (meta.locked) file.locked = true; + if (meta.chain?.id) file.chainId = meta.chain.id; + + if (!meta.timestamp) return; + const recorded = new Date(meta.timestamp); + // A sidecar with an unparsable timestamp must not produce an Invalid Date, which would + // poison every comparison it takes part in and sort unpredictably. + if (Number.isNaN(recorded.getTime())) return; + + file.backupTimestamp = recorded; + result.withTimestamp++; + + if (Math.abs(recorded.getTime() - file.lastModified.getTime()) > TIMESTAMP_DRIFT_WARNING_MS) { + result.drifted.push({ file, recorded, modified: file.lastModified }); + } +} diff --git a/src/lib/utils.ts b/src/lib/utils.ts index 98c50625..06c19bf8 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -33,6 +33,28 @@ export function formatTwoFactorCode(value: string): string { return value.replace(/\D/g, '').slice(0, 6); } +/** + * Checks whether a string names a timezone the runtime can actually resolve. + * + * Deliberately not a membership test against `Intl.supportedValuesOf('timeZone')`. That list + * holds ICU's canonical IDs, which kept the legacy tzdb names, so it carries `Asia/Calcutta` + * but not `Asia/Kolkata` and `Europe/Kiev` but not `Europe/Kyiv`. Browsers following the newer + * ECMA-402 rule offer the primary names in their picker, and a membership check rejects around + * 140 zones that every downstream consumer handles fine. Building a formatter is the honest test. + */ +export function isValidTimezone(tz: string): boolean { + // Rejects bare offsets such as "+05:30", which Intl accepts. A fixed offset carries no DST + // rules, so every schedule under it would silently shift by an hour twice a year. + if (!tz || !/^[A-Za-z]/.test(tz)) return false; + + try { + Intl.DateTimeFormat("en-US", { timeZone: tz }); + return true; + } catch { + return false; + } +} + /** * Compares two version strings (SemVer-like). * Returns 1 if v1 > v2 (v1 is newer), -1 if v1 < v2 (v1 is older), 0 if equal. diff --git a/src/services/backup/retention-service.ts b/src/services/backup/retention-service.ts index 38485f03..b6f3a1fb 100644 --- a/src/services/backup/retention-service.ts +++ b/src/services/backup/retention-service.ts @@ -1,4 +1,5 @@ import { FileInfo } from '@/lib/core/interfaces'; +import { effectiveBackupTime } from '@/lib/core/backup-files'; import { RetentionConfiguration } from '@/lib/core/retention'; import { formatInTimeZone } from 'date-fns-tz'; @@ -41,8 +42,11 @@ export class RetentionService { const lockedFiles = files.filter(f => f.locked); const processingFiles = files.filter(f => !f.locked); - // Sort files by date (newest first) - const sortedFiles = [...processingFiles].sort((a, b) => b.lastModified.getTime() - a.lastModified.getTime()); + // Sort files by date (newest first). Every tier below takes the first file it sees + // in a bucket, so this ordering is what makes "keep the newest of each bucket" true. + const sortedFiles = [...processingFiles].sort( + (a, b) => effectiveBackupTime(b).getTime() - effectiveBackupTime(a).getTime() + ); const processedFiles: FileWithReasons[] = sortedFiles.map(f => ({ file: f, keep: false, reasons: [] })); @@ -50,6 +54,11 @@ export class RetentionService { this.applySimplePolicy(processedFiles, policy.simple.keepCount); } else if (policy.mode === 'SMART' && policy.smart) { this.applySmartPolicy(processedFiles, policy.smart, timezone); + } else { + // The mode is neither NONE nor a mode with the settings it needs. Nothing marked + // a file as kept, so falling through would delete every unlocked backup on the + // destination. A policy we cannot read is a reason to keep, never to delete. + return { keep: files, delete: [], keptForChain: [] }; } // Incremental chains can only be deleted whole. A later snapshot references bytes @@ -109,13 +118,30 @@ export class RetentionService { } private static applySmartPolicy(files: FileWithReasons[], policy: NonNullable, timezone: string) { - const { daily, weekly, monthly, yearly } = policy; + // A policy written before the hourly tier existed has no value for it. 0 disables + // the tier, which is what an absent value has to mean. + const { hourly = 0, daily, weekly, monthly, yearly } = policy; - // SMART/GFS is applied as non-overlapping tiers. - // Daily picks newest unique days first. + // SMART/GFS is applied as non-overlapping tiers, finest first. + // Hourly picks newest unique hours, then Daily picks newest unique days. // Weekly/Monthly/Yearly then pick additional representatives from older buckets. + // + // Every tier only counts what it adds itself, and buckets already covered by a + // finer tier are skipped. The tiers are therefore additive rather than overlapping: + // hourly 24 with daily 7 reaches back the roughly one day the 24 hourly slots span + // plus 7 further days, not 7 days in total. restic and borg evaluate the same + // numbers as a union instead, so the totals differ for an identical config. + // // All buckets are computed in the configured timezone so that "day" aligns with - // local midnight rather than UTC midnight. + // local midnight rather than UTC midnight. The cost is that the repeated hour of a + // daylight saving change collapses into one hourly bucket once a year. + this.applyTier( + files, + hourly, + (date) => formatInTimeZone(date, timezone, 'yyyy-MM-dd-HH'), + 'Hourly' + ); + this.applyTier( files, daily, @@ -151,21 +177,24 @@ export class RetentionService { getBucketKey: (date: Date) => string, reasonPrefix: string ) { - if (limit <= 0) return; + // `!limit` catches an undefined limit, which is what a tier added after a policy was + // written looks like. `undefined <= 0` is false in JavaScript, so the bare comparison + // would let the tier run with no upper bound and keep one file per bucket forever. + if (!limit || limit <= 0) return; const usedBuckets = new Set(); // Existing keeps from earlier tiers reserve their bucket in this tier. for (const entry of files) { if (!entry.keep) continue; - usedBuckets.add(getBucketKey(entry.file.lastModified)); + usedBuckets.add(getBucketKey(effectiveBackupTime(entry.file))); } let keptInTier = 0; for (const entry of files) { if (entry.keep) continue; - const bucketKey = getBucketKey(entry.file.lastModified); + const bucketKey = getBucketKey(effectiveBackupTime(entry.file)); if (usedBuckets.has(bucketKey)) continue; entry.keep = true; diff --git a/src/services/templates/retention-policy-service.ts b/src/services/templates/retention-policy-service.ts index 2e265dbf..58551829 100644 --- a/src/services/templates/retention-policy-service.ts +++ b/src/services/templates/retention-policy-service.ts @@ -1,11 +1,29 @@ import prisma from "@/lib/prisma"; import { runBulk, type BulkResult } from "@/lib/core/bulk"; import { logger } from "@/lib/logging/logger"; -import { NotFoundError, ServiceError } from "@/lib/logging/errors"; -import type { RetentionConfiguration } from "@/lib/core/retention"; +import { NotFoundError, ServiceError, ValidationError } from "@/lib/logging/errors"; +import { RetentionConfigurationSchema, type RetentionConfiguration } from "@/lib/core/retention"; const log = logger.child({ service: "RetentionPolicyService" }); +/** + * Validates a configuration before it is stored. + * + * A retention policy is the input to file deletion, so a malformed one is not a cosmetic + * problem. Rejecting it here keeps a value written through the API from reaching the + * bucketing logic. + */ +function validateConfig(config: RetentionConfiguration): RetentionConfiguration { + try { + return RetentionConfigurationSchema.parse(config); + } catch (e) { + throw new ValidationError("Retention policy validation failed", { + field: "config", + cause: e instanceof Error ? e : undefined, + }); + } +} + export async function getRetentionPolicies() { return prisma.retentionPolicy.findMany({ orderBy: { name: "asc" } }); } @@ -28,6 +46,8 @@ export async function createRetentionPolicy(input: { throw new ServiceError("RetentionPolicyService", "createRetentionPolicy", `A retention policy named "${input.name}" already exists.`); } + const config = validateConfig(input.config); + if (input.isDefault) { await prisma.retentionPolicy.updateMany({ where: { isDefault: true }, data: { isDefault: false } }); } @@ -36,7 +56,7 @@ export async function createRetentionPolicy(input: { data: { name: input.name, description: input.description, - config: JSON.stringify(input.config), + config: JSON.stringify(config), isDefault: input.isDefault ?? false, }, }); @@ -54,6 +74,10 @@ export async function updateRetentionPolicy( isDefault?: boolean; } ) { + // Validated before anything is written, so a rejected config cannot leave the + // isDefault flag already cleared below. + const config = input.config !== undefined ? validateConfig(input.config) : undefined; + const policy = await prisma.retentionPolicy.findUnique({ where: { id } }); if (!policy) throw new NotFoundError("RetentionPolicy", id); @@ -75,8 +99,8 @@ export async function updateRetentionPolicy( data: { ...(input.name !== undefined && { name: input.name }), ...(input.description !== undefined && { description: input.description }), - ...(input.config !== undefined && { - config: JSON.stringify(input.config), + ...(config !== undefined && { + config: JSON.stringify(config), }), ...(input.isDefault !== undefined && { isDefault: input.isDefault }), }, diff --git a/tests/integration/test-configs.ts b/tests/integration/test-configs.ts index 106a3e92..9dfb34ca 100644 --- a/tests/integration/test-configs.ts +++ b/tests/integration/test-configs.ts @@ -26,6 +26,19 @@ const CLI_REQUIREMENTS: Record = { firebird: 'gbak', }; +/** + * `azure-sql` is deliberately absent from this file, and its absence is not an oversight. + * + * There is no container that behaves like Azure SQL Database. The emulator Microsoft + * published is built on azure-sql-edge, which reports EngineEdition 9 rather than 5, has + * `sys.master_files` and accepts three-part names - so a suite passing against it would + * prove the opposite of what the adapter has to handle, and the adapter's own engine guard + * would reject it on connect. + * + * Coverage is the unit suite in tests/unit/adapters/database/azure-sql/ plus a manual + * acceptance run against a real Azure SQL Database before release. + */ + // Check which CLI tools are missing const missingCli = Object.entries(CLI_REQUIREMENTS) .filter(([, cli]) => !isCliAvailable(cli)) diff --git a/tests/unit/adapters/database/azure-sql/browser.test.ts b/tests/unit/adapters/database/azure-sql/browser.test.ts new file mode 100644 index 00000000..8871609d --- /dev/null +++ b/tests/unit/adapters/database/azure-sql/browser.test.ts @@ -0,0 +1,112 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +const { mockPool, mockQuery, PoolCtor } = vi.hoisted(() => { + const mockQuery = vi.fn(); + const mockRequest = vi.fn(() => ({ query: mockQuery, input: vi.fn().mockReturnThis(), on: vi.fn() })); + const mockPool = { connect: vi.fn(), close: vi.fn(), request: mockRequest }; + const PoolCtor = vi.fn(function () { return mockPool; }); + return { mockPool, mockQuery, PoolCtor }; +}); + +vi.mock("mssql", () => ({ + default: { ConnectionPool: PoolCtor, NVarChar: "nvarchar" }, + ConnectionPool: PoolCtor, + NVarChar: "nvarchar", +})); + +import { createFakeHost } from "@/lib/testing/fake-host"; +import { getTables, getTableData } from "@/lib/adapters/database/azure-sql/browser"; + +const config = { + host: "myserver.database.windows.net", + port: 1433, + user: "backupadmin", + password: "s3cret", + database: "", + requestTimeout: 300000, +} as never; + +/** Deliberately distinctive, so a stray occurrence in a query is unmistakable. */ +const DATABASE = "zzdatabasenamezz"; + +function everyQuery(): string[] { + return mockQuery.mock.calls.map((c) => c[0] as string); +} + +function poolDatabase(): unknown { + const call = PoolCtor.mock.calls.at(-1) as unknown as [Record]; + return call[0].database; +} + +beforeEach(() => { + vi.clearAllMocks(); + mockPool.connect.mockResolvedValue(undefined); + mockPool.close.mockResolvedValue(undefined); + mockQuery.mockResolvedValue({ recordset: [] }); +}); + +/** + * Azure SQL Database rejects three-part names outright, so the database is + * selected by connecting to it. These assertions are the cheap guard against + * someone pasting a query over from the MSSQL browser, where every name is + * prefixed with `[db].` and would work nowhere here. + */ +describe("Azure SQL table browsing", () => { + it("selects the database by connecting to it, not by naming it", async () => { + await getTables(config, DATABASE, createFakeHost({ kind: "direct" })); + + expect(poolDatabase()).toBe(DATABASE); + for (const sql of everyQuery()) { + expect(sql).not.toContain(DATABASE); + } + }); + + it("reads the catalog with two-part names only", async () => { + await getTables(config, DATABASE, createFakeHost({ kind: "direct" })); + + const sql = everyQuery()[0]; + expect(sql).toContain("FROM INFORMATION_SCHEMA.TABLES"); + expect(sql).not.toContain("].INFORMATION_SCHEMA"); + }); + + it("names no database when paging through table rows either", async () => { + mockQuery.mockResolvedValue({ recordset: [{ total: 0 }] }); + + await getTableData( + config, + { database: DATABASE, table: "orders", page: 1, pageSize: 50 } as never, + createFakeHost({ kind: "direct" }), + ); + + expect(poolDatabase()).toBe(DATABASE); + for (const sql of everyQuery()) { + expect(sql).not.toContain(DATABASE); + } + }); + + it("qualifies a non-dbo table with its schema and nothing more", async () => { + mockQuery.mockResolvedValue({ recordset: [{ total: 0 }] }); + + await getTableData( + config, + { database: DATABASE, table: "sales.orders", page: 1, pageSize: 50 } as never, + createFakeHost({ kind: "direct" }), + ); + + const dataQuery = everyQuery().find((q) => q.includes("FETCH NEXT"))!; + expect(dataQuery).toContain("[sales].[orders]"); + }); + + it("escapes a closing bracket in a table name", async () => { + mockQuery.mockResolvedValue({ recordset: [{ total: 0 }] }); + + await getTableData( + config, + { database: DATABASE, table: "we]ird", page: 1, pageSize: 50 } as never, + createFakeHost({ kind: "direct" }), + ); + + const dataQuery = everyQuery().find((q) => q.includes("FETCH NEXT"))!; + expect(dataQuery).toContain("[we]]ird]"); + }); +}); diff --git a/tests/unit/adapters/database/azure-sql/connection-string.test.ts b/tests/unit/adapters/database/azure-sql/connection-string.test.ts new file mode 100644 index 00000000..c5f5aeaf --- /dev/null +++ b/tests/unit/adapters/database/azure-sql/connection-string.test.ts @@ -0,0 +1,77 @@ +import { describe, it, expect } from "vitest"; +import { + buildConnectionString, + describeConnection, +} from "@/lib/adapters/database/azure-sql/exporter/connection-string"; + +const base = { + host: "myserver.database.windows.net", + port: 1433, + user: "backupadmin", + password: "s3cret", + database: "", + requestTimeout: 300000, +}; + +describe("Azure SQL connection string", () => { + it("targets the requested database on the configured server", () => { + const cs = buildConnectionString(base as never, "shop"); + + expect(cs).toContain('Server="tcp:myserver.database.windows.net,1433"'); + expect(cs).toContain('Initial Catalog="shop"'); + expect(cs).toContain('User ID="backupadmin"'); + }); + + it("never relaxes transport security", () => { + // Both pinned rather than configurable. Azure presents a real certificate + // on every connection, so trusting an unverified one is always either a + // mistake or an interception. + const cs = buildConnectionString(base as never, "shop"); + + expect(cs).toContain('Encrypt="True"'); + expect(cs).toContain('TrustServerCertificate="False"'); + }); + + it("survives a password containing a semicolon", () => { + // The failure this prevents is silent: an unquoted `;` ends the pair, and + // the driver then reports a missing or malformed keyword rather than a bad + // password, sending people to look at the wrong thing. + const cs = buildConnectionString({ ...base, password: "pa;ss" } as never, "shop"); + + expect(cs).toContain('Password="pa;ss"'); + }); + + it("survives a password containing a double quote", () => { + const cs = buildConnectionString({ ...base, password: 'pa"ss' } as never, "shop"); + + // Doubled, which is how ADO.NET escapes the delimiter it is using. + expect(cs).toContain('Password="pa""ss"'); + }); + + it("survives a database name containing an equals sign", () => { + const cs = buildConnectionString(base as never, "a=b"); + + expect(cs).toContain('Initial Catalog="a=b"'); + }); + + it("preserves leading and trailing spaces in a password", () => { + // Unquoted, ADO.NET strips them, and the login then fails for a password + // the user can see is correct. + const cs = buildConnectionString({ ...base, password: " pw " } as never, "shop"); + + expect(cs).toContain('Password=" pw "'); + }); + + it("falls back to port 1433 when none is configured", () => { + const cs = buildConnectionString({ ...base, port: undefined } as never, "shop"); + + expect(cs).toContain('Server="tcp:myserver.database.windows.net,1433"'); + }); + + it("describes a connection without leaking the password", () => { + const description = describeConnection(base as never, "shop"); + + expect(description).toBe("myserver.database.windows.net:1433/shop as backupadmin"); + expect(description).not.toContain("s3cret"); + }); +}); diff --git a/tests/unit/adapters/database/azure-sql/connection.test.ts b/tests/unit/adapters/database/azure-sql/connection.test.ts new file mode 100644 index 00000000..6af41006 --- /dev/null +++ b/tests/unit/adapters/database/azure-sql/connection.test.ts @@ -0,0 +1,184 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +const { mockPool, mockQuery, PoolCtor } = vi.hoisted(() => { + const mockQuery = vi.fn(); + const mockRequest = vi.fn(() => ({ query: mockQuery, input: vi.fn().mockReturnThis(), on: vi.fn() })); + const mockPool = { connect: vi.fn(), close: vi.fn(), request: mockRequest }; + // A plain function, not an arrow: the adapter calls it with `new`. + const PoolCtor = vi.fn(function () { return mockPool; }); + return { mockPool, mockQuery, PoolCtor }; +}); + +vi.mock("mssql", () => ({ + default: { ConnectionPool: PoolCtor, NVarChar: "nvarchar" }, + ConnectionPool: PoolCtor, + NVarChar: "nvarchar", +})); + +import { createFakeHost } from "@/lib/testing/fake-host"; +import { test as testConnection, getDatabases } from "@/lib/adapters/database/azure-sql/connection"; +import { getDatabasesWithStats } from "@/lib/adapters/database/azure-sql/catalog"; + +const config = { + host: "myserver.database.windows.net", + port: 1433, + user: "backupadmin", + password: "s3cret", + database: "", + requestTimeout: 300000, +} as never; + +/** The connection config the driver was constructed with. */ +function lastPoolConfig(): Record { + const call = PoolCtor.mock.calls.at(-1) as unknown as [Record]; + return call[0]; +} + +function azureRow(overrides: Record = {}) { + return { ProductVersion: "12.0.2000.8", EngineEdition: 5, ServiceObjective: "S0", ...overrides }; +} + +beforeEach(() => { + vi.clearAllMocks(); + mockPool.connect.mockResolvedValue(undefined); + mockPool.close.mockResolvedValue(undefined); +}); + +describe("Azure SQL connection test", () => { + it("accepts Azure SQL Database and reports its service tier", async () => { + mockQuery.mockResolvedValue({ recordset: [azureRow()] }); + + const result = await testConnection(config, createFakeHost({ kind: "direct" })); + + expect(result.success).toBe(true); + expect(result.edition).toBe("Azure SQL Database"); + expect(result.message).toContain("S0"); + // Azure has reported this same version for years regardless of the engine + // actually running. Surfaced, never used to pick behaviour. + expect(result.version).toBe("12.0.2000"); + }); + + it.each([ + [8, /Managed Instance/], + [6, /Synapse/], + [9, /Azure SQL Edge/], + [3, /regular SQL Server instance/], + ])("refuses EngineEdition %i and names what it actually found", async (engineEdition, expected) => { + // All of these answer on 1433 with a TDS handshake and look identical until + // the first catalog query, so a bare "connection failed" would send people + // looking at firewalls. + mockQuery.mockResolvedValue({ recordset: [azureRow({ EngineEdition: engineEdition })] }); + + const result = await testConnection(config, createFakeHost({ kind: "direct" })); + + expect(result.success).toBe(false); + expect(result.message).toMatch(expected); + }); + + it("fails the test when SqlPackage is missing, and says backups are what break", async () => { + mockQuery.mockResolvedValue({ recordset: [azureRow()] }); + const host = createFakeHost({ kind: "direct", onWhich: () => null }); + + const result = await testConnection(config, host); + + expect(result.success).toBe(false); + expect(result.message).toContain("backups cannot run"); + // Still reported, because the connection itself was fine. + expect(result.edition).toBe("Azure SQL Database"); + }); + + it("points a rejected client at the firewall rule rather than the credentials", async () => { + mockPool.connect.mockRejectedValue(new Error("Client with IP address '1.2.3.4' is not allowed to access the server.")); + + const result = await testConnection(config, createFakeHost({ kind: "direct" })); + + expect(result.success).toBe(false); + expect(result.message).toContain("firewall rule"); + }); + + it("always negotiates TLS with a verified certificate", async () => { + mockQuery.mockResolvedValue({ recordset: [azureRow()] }); + + await testConnection(config, createFakeHost({ kind: "direct" })); + + const options = lastPoolConfig().options as Record; + expect(options.encrypt).toBe(true); + expect(options.trustServerCertificate).toBe(false); + }); +}); + +describe("Azure SQL database listing", () => { + it("excludes master by name, not by id", async () => { + // The MSSQL adapter filters on database_id > 4 to skip four system + // databases. Azure has only master, and assigns user database ids per + // server with no guarantee about the range. + mockQuery.mockResolvedValue({ recordset: [{ name: "shop" }, { name: "analytics" }] }); + + await getDatabases(config, createFakeHost({ kind: "direct" })); + + const sql = mockQuery.mock.calls[0][0] as string; + expect(sql).toContain("name <> 'master'"); + expect(sql).not.toContain("database_id > 4"); + }); + + it("returns nothing rather than throwing when the catalog is unreachable", async () => { + mockPool.connect.mockRejectedValue(new Error("timeout")); + + expect(await getDatabases(config, createFakeHost({ kind: "direct" }))).toEqual([]); + }); +}); + +describe("Azure SQL database stats", () => { + it("reads size and table count from inside each database", async () => { + // No three-part names exist here, so every database needs its own + // connection. sys.database_files is per database by definition. + mockQuery + .mockResolvedValueOnce({ recordset: [{ name: "shop" }] }) + .mockResolvedValueOnce({ recordset: [{ size_bytes: "8192" }] }) + .mockResolvedValueOnce({ recordset: [{ cnt: 12 }] }); + + const databases = await getDatabasesWithStats(config, createFakeHost({ kind: "direct" })); + + expect(databases).toEqual([{ name: "shop", sizeInBytes: 8192, tableCount: 12 }]); + }); + + it("keeps listing the others when one database cannot be read", async () => { + // Reproducing the MSSQL bug this adapter exists downstream of, where one + // missing catalog view took out the whole Database Explorer page, would be + // embarrassing. + mockQuery.mockResolvedValueOnce({ recordset: [{ name: "shop" }, { name: "locked" }] }); + + // The first connection is the name listing. Of the two per-database ones + // that follow, exactly one fails. Which of them is not asserted: the + // per-database reads run concurrently, so pinning the loser would be + // testing the scheduler rather than the degradation. + mockPool.connect + .mockResolvedValueOnce(undefined) + .mockRejectedValueOnce(new Error("login failed")) + .mockResolvedValue(undefined); + mockQuery.mockResolvedValue({ recordset: [{ size_bytes: "4096", cnt: 3 }] }); + + const databases = await getDatabasesWithStats(config, createFakeHost({ kind: "direct" })); + + expect(databases.map((d) => d.name).sort()).toEqual(["locked", "shop"]); + + const degraded = databases.filter((d) => d.sizeInBytes === undefined); + expect(degraded).toHaveLength(1); + expect(degraded[0].tableCount).toBe(0); + }); + + it("excludes the transaction log from the reported size", async () => { + // A BACPAC never contains the log, so counting it would overstate what a + // backup of this database is going to cost. + mockQuery + .mockResolvedValueOnce({ recordset: [{ name: "shop" }] }) + .mockResolvedValueOnce({ recordset: [{ size_bytes: "8192" }] }) + .mockResolvedValueOnce({ recordset: [{ cnt: 1 }] }); + + await getDatabasesWithStats(config, createFakeHost({ kind: "direct" })); + + const sizeQuery = mockQuery.mock.calls[1][0] as string; + expect(sizeQuery).toContain("sys.database_files"); + expect(sizeQuery).toContain("type = 0"); + }); +}); diff --git a/tests/unit/adapters/database/azure-sql/dump.test.ts b/tests/unit/adapters/database/azure-sql/dump.test.ts new file mode 100644 index 00000000..0c4e18be --- /dev/null +++ b/tests/unit/adapters/database/azure-sql/dump.test.ts @@ -0,0 +1,144 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import { join } from "node:path"; + +const { mockExport, mockGetDatabases } = vi.hoisted(() => ({ + mockExport: vi.fn(), + mockGetDatabases: vi.fn(), +})); + +vi.mock("@/lib/adapters/database/azure-sql/exporter", () => ({ + resolveExporter: () => ({ id: "sqlpackage", exportDatabase: mockExport }), +})); + +vi.mock("@/lib/adapters/database/azure-sql/connection", () => ({ + getDatabases: (...args: unknown[]) => mockGetDatabases(...args), +})); + +import { createFakeHost } from "@/lib/testing/fake-host"; +import { dump } from "@/lib/adapters/database/azure-sql/dump"; +import { isMultiDbTar, readTarManifest } from "@/lib/adapters/database/common/tar-utils"; + +let outDir: string; + +function config(database: string | string[] = "shop") { + return { + host: "myserver.database.windows.net", + port: 1433, + user: "backupadmin", + password: "s3cret", + database, + requestTimeout: 300000, + } as never; +} + +/** Stand in for SqlPackage writing the BACPAC where it was told to. */ +function exporterWritesBacpac() { + mockExport.mockImplementation(async (_cfg, dbName: string, destPath: string) => { + await writeFile(destPath, `BACPAC:${dbName}`); + }); +} + +beforeEach(async () => { + vi.clearAllMocks(); + outDir = await mkdtemp(join(os.tmpdir(), "dbackup-azure-out-")); + exporterWritesBacpac(); +}); + +afterEach(async () => { + await rm(outDir, { recursive: true, force: true }); +}); + +describe("Azure SQL dump", () => { + it("writes a single database straight to the destination", async () => { + // No TAR wrapper for one database, which is what lets the runner's + // size-polling progress watch the file it already knows about. + const out = join(outDir, "out.bacpac"); + + const result = await dump(config("shop"), out, createFakeHost({ kind: "direct" })); + + expect(result.success).toBe(true); + expect(await readFile(out, "utf8")).toBe("BACPAC:shop"); + expect(await isMultiDbTar(out)).toBe(false); + }); + + it("packs several databases with a manifest naming each one", async () => { + // A manifest, not filename parsing. Deriving names from filenames the way + // the MSSQL adapter does breaks on any database whose name contains the + // separator being parsed. + const out = join(outDir, "out.bacpac"); + + const result = await dump(config(["shop", "analytics"]), out, createFakeHost({ kind: "direct" })); + + expect(result.success).toBe(true); + expect(await isMultiDbTar(out)).toBe(true); + + const manifest = await readTarManifest(out); + expect(manifest?.sourceType).toBe("azure-sql"); + expect(manifest?.databases.map((d) => d.name).sort()).toEqual(["analytics", "shop"]); + expect(manifest?.databases.every((d) => d.format === "bacpac")).toBe(true); + }); + + it("warns about consistency before the export runs, not after it succeeds", async () => { + // A caveat that only appears once the backup worked is a caveat nobody + // reads until they have already lost something. + const messages: string[] = []; + mockExport.mockImplementation(async (_cfg, dbName: string, destPath: string) => { + messages.push("EXPORT_STARTED"); + await writeFile(destPath, `BACPAC:${dbName}`); + }); + + await dump(config("shop"), join(outDir, "out.bacpac"), createFakeHost({ kind: "direct" }), (msg) => { + messages.push(msg); + }); + + const noticeIndex = messages.findIndex((m) => m.includes("not transactionally consistent")); + expect(noticeIndex).toBeGreaterThanOrEqual(0); + expect(noticeIndex).toBeLessThan(messages.indexOf("EXPORT_STARTED")); + }); + + it("discovers every user database when the job selected none", async () => { + mockGetDatabases.mockResolvedValue(["shop", "analytics"]); + + const result = await dump(config(""), join(outDir, "out.bacpac"), createFakeHost({ kind: "direct" })); + + expect(result.success).toBe(true); + expect(mockGetDatabases).toHaveBeenCalled(); + expect(mockExport).toHaveBeenCalledTimes(2); + }); + + it("fails with a usable message when the server has no user databases", async () => { + mockGetDatabases.mockResolvedValue([]); + + const result = await dump(config(""), join(outDir, "out.bacpac"), createFakeHost({ kind: "direct" })); + + expect(result.success).toBe(false); + expect(result.error).toContain("No user databases found"); + }); + + it("reports a small export in units a reader can use", async () => { + // A BACPAC of a small database is a few kilobytes. Fixed MB reported that + // as "0.00 MB", which reads like the backup failed. + const messages: string[] = []; + + await dump(config("shop"), join(outDir, "out.bacpac"), createFakeHost({ kind: "direct" }), (msg) => { + messages.push(msg); + }); + + const line = messages.find((m) => m.startsWith("Backup finished successfully"))!; + expect(line).toContain("Bytes"); + expect(line).not.toContain("0.00 MB"); + }); + + it("reports an export that produced nothing rather than declaring success", async () => { + mockExport.mockImplementation(async (_cfg, _db, destPath: string) => { + await writeFile(destPath, ""); + }); + + const result = await dump(config("shop"), join(outDir, "out.bacpac"), createFakeHost({ kind: "direct" })); + + expect(result.success).toBe(false); + expect(result.error).toContain("empty file"); + }); +}); diff --git a/tests/unit/adapters/database/azure-sql/restore.test.ts b/tests/unit/adapters/database/azure-sql/restore.test.ts new file mode 100644 index 00000000..35b5fcd3 --- /dev/null +++ b/tests/unit/adapters/database/azure-sql/restore.test.ts @@ -0,0 +1,242 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import { join } from "node:path"; + +const { mockPool, mockQuery, PoolCtor, mockImport } = vi.hoisted(() => { + const mockQuery = vi.fn(); + const mockRequest = vi.fn(() => ({ query: mockQuery, input: vi.fn().mockReturnThis(), on: vi.fn() })); + const mockPool = { connect: vi.fn(), close: vi.fn(), request: mockRequest }; + const PoolCtor = vi.fn(function () { return mockPool; }); + return { mockPool, mockQuery, PoolCtor, mockImport: vi.fn() }; +}); + +vi.mock("mssql", () => ({ + default: { ConnectionPool: PoolCtor, NVarChar: "nvarchar" }, + ConnectionPool: PoolCtor, + NVarChar: "nvarchar", +})); + +vi.mock("@/lib/adapters/database/azure-sql/exporter", () => ({ + resolveExporter: () => ({ id: "sqlpackage", importDatabase: mockImport }), +})); + +import { createFakeHost } from "@/lib/testing/fake-host"; +import { restore } from "@/lib/adapters/database/azure-sql/restore"; +import { prepareRestore } from "@/lib/adapters/database/azure-sql/preflight"; +import { createMultiDbTar } from "@/lib/adapters/database/common/tar-utils"; + +let workDir: string; + +function config(extra: Record = {}) { + return { + host: "myserver.database.windows.net", + port: 1433, + user: "backupadmin", + password: "s3cret", + database: "shop", + requestTimeout: 300000, + ...extra, + } as never; +} + +/** Which databases importDatabase was asked to create, in call order. */ +function importedTargets(): string[] { + return mockImport.mock.calls.map((c) => c[2] as string); +} + +/** Every statement that reached the server. */ +function statements(): string[] { + return mockQuery.mock.calls.map((c) => c[0] as string); +} + +async function makeArchive(names: string[]): Promise { + const files = []; + for (const name of names) { + const path = join(workDir, `${name}.bacpac`); + await writeFile(path, `BACPAC:${name}`); + files.push({ name: `${name}.bacpac`, path, dbName: name, format: "bacpac" as const }); + } + const archive = join(workDir, "archive.tar"); + await createMultiDbTar(files, archive, { sourceType: "azure-sql" }); + return archive; +} + +beforeEach(async () => { + vi.clearAllMocks(); + workDir = await mkdtemp(join(os.tmpdir(), "dbackup-azure-restore-")); + mockPool.connect.mockResolvedValue(undefined); + mockPool.close.mockResolvedValue(undefined); + mockQuery.mockResolvedValue({ recordset: [] }); + mockImport.mockResolvedValue(undefined); +}); + +afterEach(async () => { + await rm(workDir, { recursive: true, force: true }); +}); + +describe("Azure SQL restore preflight", () => { + it("accepts a target that does not exist yet", async () => { + await expect(prepareRestore(config(), ["newdb"], createFakeHost({ kind: "direct" }))) + .resolves.toBeUndefined(); + }); + + it("accepts a target that already exists, because a restore replaces it", async () => { + // Every other adapter replaces what it restores over, and the restore + // dialog already had the user choose overwrite over rename. Refusing here + // would break a decision that was made deliberately two screens earlier. + mockQuery.mockResolvedValue({ recordset: [{ name: "shop" }] }); + + await expect(prepareRestore(config(), ["shop"], createFakeHost({ kind: "direct" }))) + .resolves.toBeUndefined(); + }); + + it("rejects a name SQL Server itself could not hold", async () => { + await expect(prepareRestore(config(), ["x".repeat(129)], createFakeHost({ kind: "direct" }))) + .rejects.toThrow(/Invalid database name/); + }); +}); + +describe("Azure SQL restore", () => { + it("imports a single BACPAC into the configured database", async () => { + const file = join(workDir, "shop.bacpac"); + await writeFile(file, "BACPAC:shop"); + + const result = await restore(config(), file, createFakeHost({ kind: "direct" })); + + expect(result.success).toBe(true); + expect(importedTargets()).toEqual(["shop"]); + }); + + it("drops an existing target first, since an import cannot overwrite in place", async () => { + const file = join(workDir, "shop.bacpac"); + await writeFile(file, "BACPAC:shop"); + mockQuery.mockResolvedValue({ recordset: [{ name: "shop" }] }); + + const result = await restore(config(), file, createFakeHost({ kind: "direct" })); + + expect(result.success).toBe(true); + expect(statements()).toContainEqual(expect.stringContaining("DROP DATABASE [shop]")); + expect(importedTargets()).toEqual(["shop"]); + }); + + it("issues no DROP when the target does not exist", async () => { + const file = join(workDir, "shop.bacpac"); + await writeFile(file, "BACPAC:shop"); + mockQuery.mockResolvedValue({ recordset: [] }); + + await restore(config(), file, createFakeHost({ kind: "direct" })); + + expect(statements().some((q) => q.includes("DROP DATABASE"))).toBe(false); + }); + + it("bracket-escapes a target name before dropping it", async () => { + // The name comes from the restore dialog, so it is user input reaching a + // statement that cannot be parameterised. + const file = join(workDir, "shop.bacpac"); + await writeFile(file, "BACPAC:shop"); + mockQuery.mockResolvedValue({ recordset: [{ name: "we]ird" }] }); + + await restore( + config({ databaseMapping: [{ originalName: "shop", targetName: "we]ird", selected: true }] }), + file, + createFakeHost({ kind: "direct" }), + ); + + expect(statements()).toContainEqual(expect.stringContaining("DROP DATABASE [we]]ird]")); + }); + + it("warns in the run log before dropping anything", async () => { + const file = join(workDir, "shop.bacpac"); + await writeFile(file, "BACPAC:shop"); + mockQuery.mockResolvedValue({ recordset: [{ name: "shop" }] }); + const logged: { msg: string; level?: string }[] = []; + + await restore(config(), file, createFakeHost({ kind: "direct" }), (msg, level) => { + logged.push({ msg, level }); + }); + + const warning = logged.find((l) => l.msg.includes("Dropping the existing database shop")); + expect(warning?.level).toBe("warning"); + // Names the way back, because Azure does keep dropped databases recoverable. + expect(warning?.msg).toContain("Deleted databases"); + }); + + it("honours a rename from the restore dialog", async () => { + const file = join(workDir, "shop.bacpac"); + await writeFile(file, "BACPAC:shop"); + + await restore( + config({ databaseMapping: [{ originalName: "shop", targetName: "shop_copy", selected: true }] }), + file, + createFakeHost({ kind: "direct" }), + ); + + expect(importedTargets()).toEqual(["shop_copy"]); + }); + + it("restores only the databases selected out of an archive", async () => { + const archive = await makeArchive(["shop", "analytics"]); + + await restore( + config({ + databaseMapping: [ + { originalName: "shop", targetName: "shop", selected: true }, + { originalName: "analytics", targetName: "analytics", selected: false }, + ], + }), + archive, + createFakeHost({ kind: "direct" }), + ); + + expect(importedTargets()).toEqual(["shop"]); + }); + + it("maps each extracted file to its own target, not to the first one", async () => { + // Matched through the manifest by filename. Pairing by array position would + // silently restore one database's contents under another's name once an + // entry has been skipped. + const archive = await makeArchive(["shop", "analytics"]); + + await restore( + config({ + databaseMapping: [ + { originalName: "shop", targetName: "shop_new", selected: true }, + { originalName: "analytics", targetName: "analytics_new", selected: true }, + ], + }), + archive, + createFakeHost({ kind: "direct" }), + ); + + expect(importedTargets().sort()).toEqual(["analytics_new", "shop_new"]); + }); + + it("connects with the privileged credentials when the restore supplies them", async () => { + // They arrive nested and are never flattened by the pipeline, so the + // adapter has to apply them itself. + const file = join(workDir, "shop.bacpac"); + await writeFile(file, "BACPAC:shop"); + + await restore( + config({ privilegedAuth: { user: "admin", password: "adminpw" } }), + file, + createFakeHost({ kind: "direct" }), + ); + + const usedConfig = mockImport.mock.calls[0][0] as { user: string; password: string }; + expect(usedConfig.user).toBe("admin"); + expect(usedConfig.password).toBe("adminpw"); + }); + + it("reports a failed import instead of claiming success", async () => { + const file = join(workDir, "shop.bacpac"); + await writeFile(file, "BACPAC:shop"); + mockImport.mockRejectedValue(new Error("Could not import package")); + + const result = await restore(config(), file, createFakeHost({ kind: "direct" })); + + expect(result.success).toBe(false); + expect(result.error).toContain("Could not import package"); + }); +}); diff --git a/tests/unit/adapters/database/azure-sql/sqlpackage.test.ts b/tests/unit/adapters/database/azure-sql/sqlpackage.test.ts new file mode 100644 index 00000000..7fecf9b9 --- /dev/null +++ b/tests/unit/adapters/database/azure-sql/sqlpackage.test.ts @@ -0,0 +1,160 @@ +import { describe, it, expect, vi } from "vitest"; +import { createFakeHost } from "@/lib/testing/fake-host"; +import { sqlpackageExporter } from "@/lib/adapters/database/azure-sql/exporter/sqlpackage"; +import { buildConnectionString } from "@/lib/adapters/database/azure-sql/exporter/connection-string"; + +const config = { + host: "myserver.database.windows.net", + port: 1433, + user: "backupadmin", + password: "s3cret", + database: "", + requestTimeout: 300000, +} as never; + +/** Collects what the adapter wrote to the run log. */ +function collector() { + const lines: { msg: string; level?: string }[] = []; + const log = (msg: string, level?: string) => lines.push({ msg, level }); + return { lines, log: log as never }; +} + +describe("SqlPackage export", () => { + it("builds the export argv SqlPackage expects", async () => { + const host = createFakeHost({ kind: "direct" }); + const { log } = collector(); + + await sqlpackageExporter.exportDatabase(config, "shop", "/tmp/shop.bacpac", host, log); + + expect(host.calls.spawn).toHaveLength(1); + const argv = host.calls.spawn[0]; + expect(argv[0]).toBe("sqlpackage"); + expect(argv).toContain("/Action:Export"); + expect(argv).toContain("/TargetFile:/tmp/shop.bacpac"); + expect(argv).toContain("/OverwriteFiles:True"); + }); + + it("passes the connection string as one argument, verbatim", async () => { + // This assertion exists to hold a documented exception in place. + // + // The adapter rules say secrets belong in options.env, never in argv. + // SqlPackage has no environment route and rejects + // /SourceConnectionString:@file, so the exception was taken deliberately, + // and it is bounded by this adapter having no SSH mode at all. + // + // If someone later moves the secret to env believing the rule was simply + // missed, this fails and points them at the reasoning in sqlpackage.ts + // rather than letting the change look correct. + const host = createFakeHost({ kind: "direct" }); + const { log } = collector(); + + await sqlpackageExporter.exportDatabase(config, "shop", "/tmp/shop.bacpac", host, log); + + const argv = host.calls.spawn[0]; + const expected = `/SourceConnectionString:${buildConnectionString(config, "shop")}`; + expect(argv).toContain(expected); + + // One argument, not split on the semicolons inside the connection string. + expect(argv.filter((a) => a.startsWith("/SourceConnectionString:"))).toHaveLength(1); + }); + + it("raises Ledger tables once, as a warning, in words a user can act on", async () => { + // SqlPackage reports this per element and buries it among the rest. The + // consequence is real: the history table and the generated-always columns + // are dropped, which is exactly the tamper evidence Ledger exists for. + const host = createFakeHost({ + kind: "direct", + onSpawn: () => ({ + stdout: [ + "Extracting schema", + "*** The ledger data in system views will not be captured in the resulting bacpac file.", + "*** Element [dbo].[x].[ledger_start_transaction_id] is a column with system-generated values in a ledger table.", + "Successfully exported database", + ].join("\n"), + }), + }); + const { lines, log } = collector(); + + await sqlpackageExporter.exportDatabase(config, "shop", "/tmp/shop.bacpac", host, log); + + const summaries = lines.filter((l) => l.msg.includes("tamper evidence is not part of this backup")); + expect(summaries).toHaveLength(1); + expect(summaries[0].level).toBe("warning"); + }); + + it("reports the cause rather than the exit code when the export fails", async () => { + // SqlPackage puts the actual reason on a *** line and then exits non-zero. + // An error carrying only the code is what made the old MSSQL failures + // unreadable. + const host = createFakeHost({ + kind: "direct", + onSpawn: () => ({ + stdout: "*** Error parsing connection string: Format of the initialization string does not conform.", + code: 1, + }), + }); + const { log } = collector(); + + await expect(sqlpackageExporter.exportDatabase(config, "shop", "/tmp/shop.bacpac", host, log)) + .rejects.toThrow(/Format of the initialization string/); + }); +}); + +describe("SqlPackage import", () => { + it("builds the import argv against the target database", async () => { + const host = createFakeHost({ kind: "direct" }); + const { log } = collector(); + + await sqlpackageExporter.importDatabase(config, "/tmp/shop.bacpac", "shop_restored", host, log); + + const argv = host.calls.spawn[0]; + expect(argv).toContain("/Action:Import"); + expect(argv).toContain("/SourceFile:/tmp/shop.bacpac"); + expect(argv).toContain(`/TargetConnectionString:${buildConnectionString(config, "shop_restored")}`); + }); + + it("reports progress only at the phases SqlPackage actually announces", async () => { + // No percentage of its own is emitted, so anything between these anchors + // would be a guess presented as a measurement. + const host = createFakeHost({ + kind: "direct", + onSpawn: () => ({ + stdout: ["Initializing deployment", "Processing Import.", "Successfully imported"].join("\n"), + }), + }); + const { log } = collector(); + const onProgress = vi.fn(); + + await sqlpackageExporter.importDatabase(config, "/tmp/x.bacpac", "shop", host, log, onProgress); + + expect(onProgress.mock.calls.map((c) => c[0])).toEqual([10, 40, 100]); + }); +}); + +describe("SqlPackage availability", () => { + it("reports the version when the binary is present", async () => { + const host = createFakeHost({ + kind: "direct", + onExec: () => ({ stdout: "170.4.83.3\n", code: 0 }), + }); + + await expect(sqlpackageExporter.probe(config, host)).resolves.toEqual({ + ok: true, + detail: "SqlPackage 170.4.83.3", + }); + }); + + it("explains a missing binary instead of throwing", async () => { + // Surfaced through the connection test, so it costs one click rather than + // a failed scheduled run at 03:00. + const host = createFakeHost({ kind: "direct", onWhich: () => null }); + + const result = await sqlpackageExporter.probe(config, host); + + expect(result.ok).toBe(false); + // Names both places it can be missing. A development checkout has no image + // at all, so blaming one there would send the reader after a phantom. + expect(result.detail).toContain("custom or outdated image"); + expect(result.detail).toContain("setup-dev-macos.sh"); + }); +}); diff --git a/tests/unit/adapters/database/mssql/connection.test.ts b/tests/unit/adapters/database/mssql/connection.test.ts index 4ea593c7..a19fa17c 100644 --- a/tests/unit/adapters/database/mssql/connection.test.ts +++ b/tests/unit/adapters/database/mssql/connection.test.ts @@ -19,6 +19,7 @@ import { test as testConnection, getDatabases, getDatabasesWithStats, + assertBackupSupported, } from "@/lib/adapters/database/mssql/connection"; import { mssqlTransport } from "@/lib/adapters/database/mssql/transport"; import type { HostKind } from "@/lib/transport/types"; @@ -124,6 +125,31 @@ describe.each(["direct", "ssh"])("MSSQL connection over a %s host", (k expect((await testConnection(baseConfig as never, mssqlHost(kind))).edition).toBe("Azure SQL Edge"); }); + it.each([ + [5, "Azure SQL Database"], + [8, "Azure SQL Managed Instance"], + ])("refuses EngineEdition %i and names the product", async (engineEdition, product) => { + // Azure answers SERVERPROPERTY('Edition') with "SQL Azure", which the + // old name parsing reduced to the meaningless "SQL". EngineEdition is + // the only reliable signal, and it has to be read before the name. + mockQuery.mockResolvedValue({ + recordset: [{ + Version: "Microsoft SQL Azure (RTM) - 12.0.2000.8", + ProductVersion: "12.0.2000.8", + Edition: "SQL Azure", + EngineEdition: engineEdition, + }], + }); + + const result = await testConnection(baseConfig as never, mssqlHost(kind)); + + expect(result.success).toBe(false); + expect(result.message).toContain(product); + expect(result.edition).toBe(product); + // Still reported, so the run log and version history stay accurate. + expect(result.version).toBe("12.0.2000"); + }); + it.each([ ["ECONNREFUSED 1.2.3.4:1433", "Connection refused"], ["Login failed for user 'sa'", "Login failed"], @@ -173,6 +199,50 @@ describe.each(["direct", "ssh"])("MSSQL connection over a %s host", (k expect(await getDatabasesWithStats(baseConfig as never, mssqlHost(kind))) .toEqual([{ name: "shop", sizeInBytes: 0, tableCount: 0 }]); }); + + it("still lists databases when sys.master_files does not exist", async () => { + // Azure SQL Database has no server-scoped catalog view. Letting that + // escape took out the whole Database Explorer with a "Connection + // Failed" card, even though the names were perfectly readable. + mockQuery + .mockRejectedValueOnce(new Error("Invalid object name 'sys.master_files'.")) + .mockResolvedValueOnce({ recordset: [{ name: "shop", state_desc: "ONLINE" }] }) + .mockResolvedValueOnce({ recordset: [{ cnt: 3 }] }); + + const databases = await getDatabasesWithStats(baseConfig as never, mssqlHost(kind)); + + expect(databases).toHaveLength(1); + expect(databases[0].name).toBe("shop"); + expect(databases[0].tableCount).toBe(3); + // Undefined, not 0. The explorer drops the column entirely rather + // than showing a table full of confident zeroes. + expect(databases[0].sizeInBytes).toBeUndefined(); + }); + }); + + describe("assertBackupSupported()", () => { + it("lets a real SQL Server through", async () => { + mockQuery.mockResolvedValue({ recordset: [{ EngineEdition: 3 }] }); + + await expect(assertBackupSupported(baseConfig as never, mssqlHost(kind))) + .resolves.toBeUndefined(); + }); + + it("rejects Azure SQL Database and says why it can never work", async () => { + mockQuery.mockResolvedValue({ recordset: [{ EngineEdition: 5 }] }); + + await expect(assertBackupSupported(baseConfig as never, mssqlHost(kind))) + .rejects.toThrow(/no BACKUP DATABASE statement/); + }); + + it("stays out of the way when the edition cannot be determined", async () => { + // Refusing on an unanswerable probe would break setups this adapter + // has always handled. Let the operation fail on its own terms. + mockPool.connect.mockRejectedValue(new Error("timeout")); + + await expect(assertBackupSupported(baseConfig as never, mssqlHost(kind))) + .resolves.toBeUndefined(); + }); }); }); diff --git a/tests/unit/adapters/database/mssql/server-paths.test.ts b/tests/unit/adapters/database/mssql/server-paths.test.ts new file mode 100644 index 00000000..59fc531b --- /dev/null +++ b/tests/unit/adapters/database/mssql/server-paths.test.ts @@ -0,0 +1,165 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +const mocks = vi.hoisted(() => ({ executeQuery: vi.fn() })); + +vi.mock("@/lib/adapters/database/mssql/connection", () => ({ + executeQuery: (...args: unknown[]) => mocks.executeQuery(...args), +})); + +import { + buildMoveTargets, + getInstanceDefaultPaths, + joinServerPath, + serverDirname, + type RestoreFileEntry, +} from "@/lib/adapters/database/mssql/server-paths"; +import { createFakeHost } from "@/lib/testing/fake-host"; +import type { MSSQLConfig } from "@/lib/adapters/definitions"; + +const config = { host: "sql.example.com", port: 1433, user: "dbackup" } as MSSQLConfig; + +function dataFile(overrides: Partial = {}): RestoreFileEntry { + return { logicalName: "Shop", type: "D", physicalName: "/var/opt/mssql/data/Shop.mdf", ...overrides }; +} + +function logFile(overrides: Partial = {}): RestoreFileEntry { + return { logicalName: "Shop_log", type: "L", physicalName: "/var/opt/mssql/data/Shop_log.ldf", ...overrides }; +} + +describe("joinServerPath", () => { + it("leaves POSIX paths exactly as path.posix.join produced them", () => { + // Every existing source has a POSIX backup path, so this branch has to + // stay byte for byte identical or every stored config changes behaviour. + expect(joinServerPath("/var/opt/mssql/backup", "shop.bak")).toBe("/var/opt/mssql/backup/shop.bak"); + expect(joinServerPath("/var/opt/mssql/backup/", "shop.bak")).toBe("/var/opt/mssql/backup/shop.bak"); + }); + + it("uses backslashes for a Windows drive path", () => { + expect(joinServerPath("D:\\SQLBackup", "shop.bak")).toBe("D:\\SQLBackup\\shop.bak"); + expect(joinServerPath("D:\\SQLBackup\\", "shop.bak")).toBe("D:\\SQLBackup\\shop.bak"); + }); + + it("keeps a UNC share intact", () => { + expect(joinServerPath("\\\\synology-nas\\sql-backup", "shop.bak")).toBe("\\\\synology-nas\\sql-backup\\shop.bak"); + }); + + it("keeps forward slashes on a drive path that was written with them", () => { + // A drive letter does not decide the separator. `D:/SQLBackup` is the + // spelling that also works over SFTP, where a backslash is an ordinary + // character, so it has to survive untouched. + expect(joinServerPath("D:/SQLBackup", "shop.bak")).toBe("D:/SQLBackup/shop.bak"); + }); +}); + +describe("serverDirname", () => { + it("returns the directory of a POSIX path", () => { + expect(serverDirname("/var/opt/mssql/data/Shop.mdf")).toBe("/var/opt/mssql/data"); + }); + + it("returns the directory of a Windows path", () => { + expect(serverDirname("D:\\SQL\\DATA\\Shop.mdf")).toBe("D:\\SQL\\DATA"); + expect(serverDirname("\\\\nas\\share\\Shop.mdf")).toBe("\\\\nas\\share"); + }); + + it("keeps a filesystem root, which is its own separator", () => { + expect(serverDirname("/Shop.mdf")).toBe("/"); + }); + + it("returns null for a bare file name", () => { + expect(serverDirname("Shop.mdf")).toBeNull(); + }); +}); + +describe("buildMoveTargets", () => { + it("places files in the instance default directory on a Windows server", () => { + // The regression: this used to be a hardcoded /var/opt/mssql/data, which a + // Windows server resolves against the current drive and rejects with + // operating system error 3. + const targets = buildMoveTargets([dataFile(), logFile()], "Shop_Copy", { + data: "D:\\SQL\\DATA\\", + log: "E:\\SQL\\LOGS\\", + }); + + expect(targets).toEqual([ + { logicalName: "Shop", physicalPath: "D:\\SQL\\DATA\\Shop_Copy.mdf" }, + { logicalName: "Shop_log", physicalPath: "E:\\SQL\\LOGS\\Shop_Copy.ldf" }, + ]); + }); + + it("falls back to the directory the backup came from", () => { + // SQL Server 2008 R2 has no InstanceDefaultDataPath, and some instances + // answer NULL for it. The file's own directory is right whenever the + // restore targets the server that wrote the backup. + const targets = buildMoveTargets( + [dataFile({ physicalName: "D:\\SQL\\DATA\\Shop.mdf" }), logFile({ physicalName: "E:\\SQL\\LOGS\\Shop_log.ldf" })], + "Shop_Copy", + {}, + ); + + expect(targets).toEqual([ + { logicalName: "Shop", physicalPath: "D:\\SQL\\DATA\\Shop_Copy.mdf" }, + { logicalName: "Shop_log", physicalPath: "E:\\SQL\\LOGS\\Shop_Copy.ldf" }, + ]); + }); + + it("still places files correctly on a Linux server", () => { + const targets = buildMoveTargets([dataFile(), logFile()], "Shop_Copy", { data: "/var/opt/mssql/data/" }); + + expect(targets).toEqual([ + { logicalName: "Shop", physicalPath: "/var/opt/mssql/data/Shop_Copy.mdf" }, + { logicalName: "Shop_log", physicalPath: "/var/opt/mssql/data/Shop_Copy.ldf" }, + ]); + }); + + it("gives every data file its own name", () => { + // Both used to be moved onto the same .mdf, which SQL Server refuses. + const targets = buildMoveTargets( + [dataFile(), dataFile({ logicalName: "Shop_2", physicalName: "/var/opt/mssql/data/Shop_2.ndf" }), logFile()], + "Shop_Copy", + { data: "/var/opt/mssql/data" }, + ); + + expect(targets.map((t) => t.physicalPath)).toEqual([ + "/var/opt/mssql/data/Shop_Copy.mdf", + "/var/opt/mssql/data/Shop_Copy_2.ndf", + "/var/opt/mssql/data/Shop_Copy.ldf", + ]); + }); + + it("rejects a rename it cannot place, naming the way out", () => { + const filestream = dataFile({ logicalName: "Shop_fs", type: "S", physicalName: "/var/opt/mssql/data/Shop_fs" }); + + expect(() => buildMoveTargets([filestream], "Shop_Copy", { data: "/var/opt/mssql/data" })) + .toThrow(/original database name/); + }); +}); + +describe("getInstanceDefaultPaths", () => { + beforeEach(() => { + mocks.executeQuery.mockReset(); + }); + + it("reads both directories from the instance", async () => { + mocks.executeQuery.mockResolvedValue({ + recordset: [{ DataPath: "D:\\SQL\\DATA\\", LogPath: "E:\\SQL\\LOGS\\" }], + }); + + const paths = await getInstanceDefaultPaths(config, createFakeHost({ kind: "direct" })); + + expect(paths).toEqual({ data: "D:\\SQL\\DATA\\", log: "E:\\SQL\\LOGS\\" }); + }); + + it("reports nothing when the server answers NULL", async () => { + // SERVERPROPERTY answers NULL for a property the server has never heard + // of, so a pre-2012 instance lands here rather than in the catch. + mocks.executeQuery.mockResolvedValue({ recordset: [{ DataPath: null, LogPath: null }] }); + + expect(await getInstanceDefaultPaths(config, createFakeHost({ kind: "direct" }))).toEqual({}); + }); + + it("reports nothing when the query fails", async () => { + mocks.executeQuery.mockRejectedValue(new Error("permission denied")); + + expect(await getInstanceDefaultPaths(config, createFakeHost({ kind: "direct" }))).toEqual({}); + }); +}); diff --git a/tests/unit/adapters/mssql/dump.test.ts b/tests/unit/adapters/mssql/dump.test.ts index 26ade96b..effbccda 100644 --- a/tests/unit/adapters/mssql/dump.test.ts +++ b/tests/unit/adapters/mssql/dump.test.ts @@ -3,13 +3,15 @@ import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import { join } from "node:path"; -const { mockExecuteWithMessages, mockGetDatabases, mockSupportsCompression } = vi.hoisted(() => ({ +const { mockExecuteWithMessages, mockGetDatabases, mockSupportsCompression, mockAssertSupported } = vi.hoisted(() => ({ mockExecuteWithMessages: vi.fn(), mockGetDatabases: vi.fn(), mockSupportsCompression: vi.fn(), + mockAssertSupported: vi.fn(), })); vi.mock("@/lib/adapters/database/mssql/connection", () => ({ + assertBackupSupported: (...args: unknown[]) => mockAssertSupported(...args), executeQueryWithMessages: (...args: unknown[]) => mockExecuteWithMessages(...args), getDatabases: (...args: unknown[]) => mockGetDatabases(...args), supportsCompression: (...args: unknown[]) => mockSupportsCompression(...args), @@ -61,6 +63,7 @@ beforeEach(async () => { mountDir = await mkdtemp(join(os.tmpdir(), "dbackup-mssql-mount-")); outDir = await mkdtemp(join(os.tmpdir(), "dbackup-mssql-out-")); mockSupportsCompression.mockResolvedValue(true); + mockAssertSupported.mockResolvedValue(undefined); mockExecuteWithMessages.mockResolvedValue({ result: {}, messages: [] }); }); @@ -69,6 +72,24 @@ afterEach(async () => { await rm(outDir, { recursive: true, force: true }); }); +describe("MSSQL dump against an unsupported engine", () => { + it("refuses before sending a single statement", async () => { + // Azure SQL Database used to get all the way to BACKUP DATABASE and fail + // there with "not supported in this version of SQL Server", which names + // neither the product nor the way forward. + serverWritesBackup(); + mockAssertSupported.mockRejectedValue( + new Error("Azure SQL Database is not supported by this adapter. It has no BACKUP DATABASE statement at all."), + ); + + const result = await dump(config() as never, join(outDir, "out.bak"), createFakeHost({ kind: "direct" })); + + expect(result.success).toBe(false); + expect(result.error).toContain("Azure SQL Database is not supported"); + expect(queries()).toHaveLength(0); + }); +}); + describe("MSSQL dump with a shared mount", () => { beforeEach(serverWritesBackup); diff --git a/tests/unit/adapters/mssql/restore.test.ts b/tests/unit/adapters/mssql/restore.test.ts index 73d86592..47356260 100644 --- a/tests/unit/adapters/mssql/restore.test.ts +++ b/tests/unit/adapters/mssql/restore.test.ts @@ -3,13 +3,15 @@ import { mkdtemp, readdir, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import { join } from "node:path"; -const { mockExecuteQuery, mockExecuteWithMessages, mockExecuteParameterized } = vi.hoisted(() => ({ +const { mockExecuteQuery, mockExecuteWithMessages, mockExecuteParameterized, mockAssertSupported } = vi.hoisted(() => ({ mockExecuteQuery: vi.fn(), mockExecuteWithMessages: vi.fn(), mockExecuteParameterized: vi.fn(), + mockAssertSupported: vi.fn(), })); vi.mock("@/lib/adapters/database/mssql/connection", () => ({ + assertBackupSupported: (...args: unknown[]) => mockAssertSupported(...args), executeQuery: (...args: unknown[]) => mockExecuteQuery(...args), executeQueryWithMessages: (...args: unknown[]) => mockExecuteWithMessages(...args), executeParameterizedQuery: (...args: unknown[]) => mockExecuteParameterized(...args), @@ -44,6 +46,7 @@ beforeEach(async () => { mountDir = await mkdtemp(join(os.tmpdir(), "dbackup-mssql-mount-")); srcDir = await mkdtemp(join(os.tmpdir(), "dbackup-mssql-src-")); + mockAssertSupported.mockResolvedValue(undefined); mockExecuteParameterized.mockResolvedValue({ recordset: [{ state_desc: "ONLINE" }] }); mockExecuteQuery.mockResolvedValue({ recordset: [ @@ -84,6 +87,16 @@ describe("MSSQL prepareRestore", () => { await expect(prepareRestore(config() as never, ["newdb"], createFakeHost({ kind: "direct" }))) .resolves.toBeUndefined(); }); + + it("refuses an unsupported engine before inspecting any target", async () => { + // Preflight runs before an Execution row exists, so refusing here leaves + // no failed run behind for the user to interpret. + mockAssertSupported.mockRejectedValue(new Error("Azure SQL Managed Instance is not supported.")); + + await expect(prepareRestore(config() as never, ["testdb"], createFakeHost({ kind: "direct" }))) + .rejects.toThrow("Azure SQL Managed Instance is not supported"); + expect(mockExecuteParameterized).not.toHaveBeenCalled(); + }); }); describe("MSSQL restore with a shared mount", () => { diff --git a/tests/unit/adapters/s3-upload-tuning.test.ts b/tests/unit/adapters/s3-upload-tuning.test.ts new file mode 100644 index 00000000..37f7c3f0 --- /dev/null +++ b/tests/unit/adapters/s3-upload-tuning.test.ts @@ -0,0 +1,243 @@ +import { describe, it, expect } from "vitest"; +import { + resolveS3UploadTuning, + s3UploadTuningRange, + s3UploadMemoryBudget, + DEFAULT_S3_UPLOAD_TUNING, + S3_MIN_PART_SIZE_MB, + S3_MAX_PARTS, +} from "@/lib/adapters/s3-upload-tuning"; +import { ADAPTER_DEFINITIONS } from "@/lib/adapters/definitions"; + +const MB = 1024 * 1024; +const S3_ADAPTERS = ["s3-aws", "s3-generic", "s3-r2", "s3-hetzner"]; + +describe("S3 upload tuning on the config schemas", () => { + it.each(S3_ADAPTERS)("keeps both tuning values through validation: %s", (id) => { + // The connection form validates with zodResolver, and Zod drops keys the schema does not + // declare. A field missing here is not a typing detail: the value is discarded in the + // browser before the request is sent, so saving 16 silently stores nothing and the form + // shows the default again when reopened. + const def = ADAPTER_DEFINITIONS.find((d) => d.id === id)!; + const parsed = def.configSchema.partial().parse({ + uploadConcurrency: 16, + uploadPartSizeMb: 32, + }) as Record; + expect(parsed.uploadConcurrency).toBe(16); + expect(parsed.uploadPartSizeMb).toBe(32); + }); + + it("does not put the fields on an adapter that uploads as a single stream", () => { + // Stored on a WebDAV or SFTP connection they would never be read, which reads to the + // user as a setting that does nothing. + const def = ADAPTER_DEFINITIONS.find((d) => d.id === "webdav")!; + const parsed = def.configSchema.partial().parse({ uploadConcurrency: 16 }) as Record; + expect(parsed.uploadConcurrency).toBeUndefined(); + }); +}); + +describe("s3UploadTuningRange", () => { + it("gives every S3 adapter the shared range", () => { + for (const id of S3_ADAPTERS) { + expect(s3UploadTuningRange(id)).toEqual(DEFAULT_S3_UPLOAD_TUNING); + } + }); + + it("returns nothing for an adapter that does not upload in parts", () => { + // The form hides the field on this rather than showing a disabled 1. + expect(s3UploadTuningRange("sftp")).toBeUndefined(); + expect(s3UploadTuningRange("does-not-exist")).toBeUndefined(); + }); +}); + +describe("resolveS3UploadTuning", () => { + it("beats the AWS SDK's own defaults, which is the entire point", () => { + // The SDK uses 4 parts of 5 MB when neither is given. Measured against R2 over a + // 10 Gbit link that is 27 MB/s, while the same run hashed the file locally at 460 MB/s. + const { queueSize, partSize } = resolveS3UploadTuning("s3-r2", {}); + expect(queueSize).toBeGreaterThan(4); + expect(partSize).toBeGreaterThan(5 * MB); + }); + + it("uses the adapter default when the connection names no value", () => { + expect(resolveS3UploadTuning("s3-aws", {})).toEqual({ + queueSize: DEFAULT_S3_UPLOAD_TUNING.concurrency.default, + partSize: DEFAULT_S3_UPLOAD_TUNING.partSizeMb.default * MB, + adjustment: 'none', + }); + expect(resolveS3UploadTuning("s3-aws", undefined).queueSize).toBe( + DEFAULT_S3_UPLOAD_TUNING.concurrency.default + ); + }); + + it("uses the values the connection stored", () => { + const { queueSize, partSize } = resolveS3UploadTuning("s3-r2", { + uploadConcurrency: 24, + uploadPartSizeMb: 32, + }); + expect(queueSize).toBe(24); + expect(partSize).toBe(32 * MB); + }); + + it("accepts the values as strings, as a form or an imported config can store them", () => { + const { queueSize, partSize } = resolveS3UploadTuning("s3-r2", { + uploadConcurrency: "16", + uploadPartSizeMb: "16", + }); + expect(queueSize).toBe(16); + expect(partSize).toBe(16 * MB); + }); + + it("clamps values above the ceiling", () => { + // These arrive from JSON that a restored export or a hand-edited database can put + // anything into, and unbounded they multiply straight into memory. + const { queueSize, partSize } = resolveS3UploadTuning("s3-aws", { + uploadConcurrency: 5000, + uploadPartSizeMb: 5000, + }); + expect(queueSize).toBe(DEFAULT_S3_UPLOAD_TUNING.concurrency.max); + expect(partSize).toBe(DEFAULT_S3_UPLOAD_TUNING.partSizeMb.max * MB); + }); + + it("never produces a part S3 would refuse", () => { + // S3 rejects any part below 5 MB except the last, so a stored 1 fails the upload + // outright rather than making it slower. + const { partSize } = resolveS3UploadTuning("s3-aws", { uploadPartSizeMb: 1 }); + expect(partSize).toBe(S3_MIN_PART_SIZE_MB * MB); + }); + + it("never resolves below one part in flight", () => { + expect(resolveS3UploadTuning("s3-aws", { uploadConcurrency: 0 }).queueSize).toBe(1); + expect(resolveS3UploadTuning("s3-aws", { uploadConcurrency: -5 }).queueSize).toBe(1); + }); + + it("falls back rather than guessing when a stored value is not a number", () => { + const { queueSize, partSize } = resolveS3UploadTuning("s3-aws", { + uploadConcurrency: "lots", + uploadPartSizeMb: null, + }); + expect(queueSize).toBe(DEFAULT_S3_UPLOAD_TUNING.concurrency.default); + expect(partSize).toBe(DEFAULT_S3_UPLOAD_TUNING.partSizeMb.default * MB); + }); + + it("raises the part size so a large archive stays under S3's part limit", () => { + // Passing an explicit partSize switches off the SDK's own max(5 MB, size / 10000), and + // with it the only thing keeping this upload legal. A 500 GB backup at the default 8 MB + // would be 64.000 parts, which S3 rejects outright. + const fiveHundredGb = 500 * 1024 * MB; + const { partSize, adjustment } = resolveS3UploadTuning("s3-aws", {}, fiveHundredGb); + + expect(adjustment).toBe('raised-for-part-limit'); + expect(partSize).toBeGreaterThan(DEFAULT_S3_UPLOAD_TUNING.partSizeMb.default * MB); + expect(Math.ceil(fiveHundredGb / partSize)).toBeLessThanOrEqual(S3_MAX_PARTS); + }); + + it("leaves the configured part size alone for an archive that fits", () => { + const { partSize, adjustment } = resolveS3UploadTuning("s3-aws", {}, 2 * 1024 * MB); + expect(adjustment).toBe('none'); + expect(partSize).toBe(DEFAULT_S3_UPLOAD_TUNING.partSizeMb.default * MB); + }); + + it("keeps the configured size when the archive size is unknown", () => { + // The metadata sidecars upload through the same path, and a stat can fail. + expect(resolveS3UploadTuning("s3-aws", {}, undefined).partSize).toBe( + DEFAULT_S3_UPLOAD_TUNING.partSizeMb.default * MB + ); + expect(resolveS3UploadTuning("s3-aws", {}, 0).partSize).toBe( + DEFAULT_S3_UPLOAD_TUNING.partSizeMb.default * MB + ); + }); +}); + +describe("resolveS3UploadTuning keeps every connection fed", () => { + it("lowers the part size when the archive would not split into enough parts", () => { + // The case measured against R2: a 1.39 GB archive at 32 parts of 64 MB is only 21 parts, + // so 11 connections never received one and the upload took as long as a single part. + // 123 MB/s that way, 187 MB/s once every connection had work. + const archive = 1_387_008_512; + const { partSize, adjustment } = resolveS3UploadTuning( + "s3-r2", + { uploadConcurrency: 32, uploadPartSizeMb: 64 }, + archive + ); + + expect(adjustment).toBe('lowered-to-fill-parallelism'); + expect(partSize).toBeLessThan(64 * MB); + expect(Math.ceil(archive / partSize)).toBeGreaterThanOrEqual(32); + }); + + it("gives every connection more than one part, so a slow one does not set the pace", () => { + // At exactly one round the upload finishes when its slowest part does, with nothing + // behind it to absorb a connection that stalls. + const archive = 1_387_008_512; + const { partSize } = resolveS3UploadTuning( + "s3-r2", + { uploadConcurrency: 32, uploadPartSizeMb: 64 }, + archive + ); + expect(Math.ceil(archive / partSize)).toBeGreaterThanOrEqual(64); + }); + + it("never raises the part size to fill connections, only lowers it", () => { + // The stored value is a memory ceiling. Growing past it to reach some ideal part count + // would spend memory the user did not agree to. + const { partSize, adjustment } = resolveS3UploadTuning( + "s3-r2", + { uploadConcurrency: 4, uploadPartSizeMb: 8 }, + 500 * 1024 * MB + ); + expect(adjustment).not.toBe('lowered-to-fill-parallelism'); + expect(partSize).toBeGreaterThanOrEqual(8 * MB); + }); + + it("stops at S3's 5 MB minimum for an archive too small to fill every connection", () => { + // A 20 MB backup cannot keep 32 connections busy at any legal part size. Splitting + // further would trade a valid upload for parallelism it can never reach. + const { partSize } = resolveS3UploadTuning( + "s3-r2", + { uploadConcurrency: 32, uploadPartSizeMb: 64 }, + 20 * MB + ); + expect(partSize).toBe(S3_MIN_PART_SIZE_MB * MB); + }); + + it("lets the part limit win over the ceiling, because the alternative is a rejected upload", () => { + const eightHundredGb = 800 * 1024 * MB; + const { partSize, adjustment } = resolveS3UploadTuning( + "s3-aws", + { uploadConcurrency: 32, uploadPartSizeMb: 16 }, + eightHundredGb + ); + + expect(adjustment).toBe('raised-for-part-limit'); + expect(partSize).toBeGreaterThan(16 * MB); + expect(Math.ceil(eightHundredGb / partSize)).toBeLessThanOrEqual(S3_MAX_PARTS); + }); + + it("leaves a ceiling alone that already fits the archive", () => { + // Manu's measured optimum: 32 parts of 16 MB on a 1.39 GB archive, 187 MB/s. + const { partSize, adjustment } = resolveS3UploadTuning( + "s3-r2", + { uploadConcurrency: 32, uploadPartSizeMb: 16 }, + 1_387_008_512 + ); + expect(adjustment).toBe('none'); + expect(partSize).toBe(16 * MB); + }); +}); + +describe("s3UploadMemoryBudget", () => { + it("counts the parts in flight plus the one being filled", () => { + expect(s3UploadMemoryBudget(8, 8)).toBe(9 * 8 * MB); + }); + + it("keeps the shipped default inside what a small container can spare", () => { + // The default has to work in a 512 MB container, which is where most self-hosted + // installations run. The ceiling is opt-in and the form states its cost. + const shipped = s3UploadMemoryBudget( + DEFAULT_S3_UPLOAD_TUNING.concurrency.default, + DEFAULT_S3_UPLOAD_TUNING.partSizeMb.default + ); + expect(shipped).toBeLessThan(128 * MB); + }); +}); diff --git a/tests/unit/adapters/storage/s3-upload-parts.test.ts b/tests/unit/adapters/storage/s3-upload-parts.test.ts new file mode 100644 index 00000000..50ed0bcb --- /dev/null +++ b/tests/unit/adapters/storage/s3-upload-parts.test.ts @@ -0,0 +1,170 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { Readable } from "stream"; +import { Upload } from "@aws-sdk/lib-storage"; +import { + S3GenericAdapter, + S3AWSAdapter, + S3R2Adapter, + S3HetznerAdapter, +} from "@/lib/adapters/storage/s3"; +import { DEFAULT_S3_UPLOAD_TUNING } from "@/lib/adapters/s3-upload-tuning"; + +const MB = 1024 * 1024; + +const { mockUploadDone, mockStat } = vi.hoisted(() => ({ + mockUploadDone: vi.fn(), + mockStat: vi.fn(), +})); + +vi.mock("@aws-sdk/client-s3", () => ({ + S3Client: vi.fn(function (this: Record) { + this.send = vi.fn().mockResolvedValue({}); + this.destroy = vi.fn(); + }), + ListObjectsV2Command: vi.fn(), + GetObjectCommand: vi.fn(), + DeleteObjectCommand: vi.fn(), + PutObjectCommand: vi.fn(), + HeadObjectCommand: vi.fn(), + HeadBucketCommand: vi.fn(), + StorageClass: {}, +})); + +vi.mock("@aws-sdk/lib-storage", () => ({ + Upload: vi.fn(function (this: Record) { + this.on = vi.fn(); + this.done = mockUploadDone; + }), +})); + +vi.mock("fs", () => { + const mod = { + createReadStream: vi.fn(() => Readable.from(["data"])), + createWriteStream: vi.fn(), + }; + return { ...mod, default: mod }; +}); + +vi.mock("fs/promises", () => { + const mod = { stat: mockStat }; + return { ...mod, default: mod }; +}); + +vi.mock("@/lib/logging/logger", () => ({ + logger: { child: () => ({ info: vi.fn(), error: vi.fn(), warn: vi.fn(), debug: vi.fn() }) }, +})); + +vi.mock("@/lib/logging/errors", () => ({ wrapError: (e: unknown) => e })); + +const genericConfig = { + endpoint: "https://s3.example.com", + region: "us-east-1", + bucket: "my-bucket", + accessKeyId: "KEY", + secretAccessKey: "SECRET", + pathPrefix: "", +}; + +/** The options `new Upload()` was constructed with on the most recent call. */ +function lastUploadOptions() { + return (Upload as unknown as ReturnType).mock.calls.at(-1)?.[0]; +} + +describe("S3 multipart upload tuning reaches the SDK", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockUploadDone.mockResolvedValue({}); + mockStat.mockResolvedValue({ size: 2 * 1024 * MB }); + }); + + it("uploads with DBackup's defaults rather than the SDK's 4 parts of 5 MB", async () => { + await S3GenericAdapter.upload(genericConfig as never, "/tmp/backup.tar", "Job/backup.tar"); + + const options = lastUploadOptions(); + expect(options.queueSize).toBe(DEFAULT_S3_UPLOAD_TUNING.concurrency.default); + expect(options.partSize).toBe(DEFAULT_S3_UPLOAD_TUNING.partSizeMb.default * MB); + }); + + it("uses what the connection stored", async () => { + await S3GenericAdapter.upload( + { ...genericConfig, uploadConcurrency: 24, uploadPartSizeMb: 32 } as never, + "/tmp/backup.tar", + "Job/backup.tar" + ); + + const options = lastUploadOptions(); + expect(options.queueSize).toBe(24); + expect(options.partSize).toBe(32 * MB); + }); + + it("clamps a stored value that would otherwise multiply straight into memory", async () => { + // Large enough that the ceiling is what limits the part size rather than the derivation + // that keeps every connection fed, so this asserts the clamp and nothing else. + mockStat.mockResolvedValue({ size: 16 * 1024 * MB }); + + await S3GenericAdapter.upload( + { ...genericConfig, uploadConcurrency: 999, uploadPartSizeMb: 999 } as never, + "/tmp/backup.tar", + "Job/backup.tar" + ); + + const options = lastUploadOptions(); + expect(options.queueSize).toBe(DEFAULT_S3_UPLOAD_TUNING.concurrency.max); + expect(options.partSize).toBe(DEFAULT_S3_UPLOAD_TUNING.partSizeMb.max * MB); + }); + + it("lowers the part size when the archive would leave connections without one", async () => { + // The R2 measurement: 1.39 GB at 32 parts of 64 MB is 21 parts, so 11 connections idled + // and the upload took as long as one part. + const archive = 1_387_008_512; + mockStat.mockResolvedValue({ size: archive }); + + await S3GenericAdapter.upload( + { ...genericConfig, uploadConcurrency: 32, uploadPartSizeMb: 64 } as never, + "/tmp/backup.tar", + "Job/backup.tar" + ); + + const { queueSize, partSize } = lastUploadOptions(); + expect(partSize).toBeLessThan(64 * MB); + expect(Math.ceil(archive / partSize)).toBeGreaterThanOrEqual(queueSize); + }); + + it("raises the part size so a large archive stays under S3's 10,000-part limit", async () => { + const size = 500 * 1024 * MB; + mockStat.mockResolvedValue({ size }); + + await S3GenericAdapter.upload(genericConfig as never, "/tmp/huge.tar", "Job/huge.tar"); + + const { partSize } = lastUploadOptions(); + expect(Math.ceil(size / partSize)).toBeLessThanOrEqual(10_000); + }); + + it("still uploads when the file cannot be stat'd", async () => { + // The size only picks the part size. Losing it must not lose the backup. + mockStat.mockRejectedValue(new Error("ENOENT")); + + const result = await S3GenericAdapter.upload(genericConfig as never, "/tmp/x.tar", "Job/x.tar"); + + expect(result).toBe(true); + expect(lastUploadOptions().partSize).toBe(DEFAULT_S3_UPLOAD_TUNING.partSizeMb.default * MB); + }); + + it.each([ + ["S3AWSAdapter", S3AWSAdapter, { region: "us-east-1", bucket: "b", accessKeyId: "K", secretAccessKey: "S" }], + ["S3R2Adapter", S3R2Adapter, { accountId: "abc", bucket: "b", accessKeyId: "K", secretAccessKey: "S" }], + ["S3HetznerAdapter", S3HetznerAdapter, { region: "fsn1", bucket: "b", accessKeyId: "K", secretAccessKey: "S" }], + ])("wires the stored values through on %s", async (_name, adapter, config) => { + // One adapter missing its wiring would silently fall back to the defaults, which is the + // one failure this change cannot notice by itself. + await adapter.upload( + { ...config, uploadConcurrency: 12, uploadPartSizeMb: 16 } as never, + "/tmp/backup.tar", + "Job/backup.tar" + ); + + const options = lastUploadOptions(); + expect(options.queueSize).toBe(12); + expect(options.partSize).toBe(16 * MB); + }); +}); diff --git a/tests/unit/adapters/storage/s3.test.ts b/tests/unit/adapters/storage/s3.test.ts index 975f0a11..cca9fc92 100644 --- a/tests/unit/adapters/storage/s3.test.ts +++ b/tests/unit/adapters/storage/s3.test.ts @@ -220,18 +220,67 @@ describe("S3 Adapters - shared logic via S3GenericAdapter", () => { await expect(S3GenericAdapter.list(genericConfig, "Job")).rejects.toThrow("Access Denied"); }); - it("filters out zero-size entries (virtual folder markers)", async () => { + // A folder marker is a key ending in "/". Recognising them by size instead threw away + // every genuine empty file with them, which left an empty file out of a directory + // backup and made it read as deleted everywhere the listing decides what exists. + it("drops folder markers but keeps a genuinely empty file", async () => { mockSend.mockResolvedValue({ Contents: [ { Key: "Job/", Size: 0, LastModified: new Date() }, + { Key: "Job/empty.txt", Size: 0, LastModified: new Date() }, { Key: "Job/backup.sql", Size: 512, LastModified: new Date() }, ], }); const result = await S3GenericAdapter.list(genericConfig, "Job"); + expect(result.map((f) => f.name)).toEqual(["empty.txt", "backup.sql"]); + expect(result[0].size).toBe(0); + }); + + // A marker's basename is the folder name ("backups/foo/" -> "foo"), so the trailing + // slash has to be tested on the raw key, before basename and before the prefix strip. + it("drops a nested folder marker whose basename looks like a file", async () => { + mockSend.mockResolvedValue({ + Contents: [{ Key: "Job/nested/", Size: 0, LastModified: new Date() }], + }); + + expect(await S3GenericAdapter.list(genericConfig, "Job")).toEqual([]); + }); + + // S3 answers with at most 1000 keys per page, in lexicographic order. Reading only the + // first page returned the alphabetically first keys, which for timestamped backup + // names are the oldest - so every recent backup was invisible, silently. + it("follows the continuation token until the listing is complete", async () => { + mockSend + .mockResolvedValueOnce({ + Contents: [{ Key: "Job/a.sql", Size: 10, LastModified: new Date() }], + IsTruncated: true, + NextContinuationToken: "page-2", + }) + .mockResolvedValueOnce({ + Contents: [{ Key: "Job/b.sql", Size: 20, LastModified: new Date() }], + IsTruncated: false, + }); + + const result = await S3GenericAdapter.list(genericConfig, "Job"); + + expect(result.map((f) => f.name)).toEqual(["a.sql", "b.sql"]); + expect(mockSend).toHaveBeenCalledTimes(2); + expect(mockSend.mock.calls[0][0].ContinuationToken).toBeUndefined(); + expect(mockSend.mock.calls[1][0].ContinuationToken).toBe("page-2"); + }); + + it("stops after a truncated page that hands back no token", async () => { + mockSend.mockResolvedValue({ + Contents: [{ Key: "Job/a.sql", Size: 10, LastModified: new Date() }], + IsTruncated: true, + }); + + const result = await S3GenericAdapter.list(genericConfig, "Job"); + expect(result).toHaveLength(1); - expect(result[0].name).toBe("backup.sql"); + expect(mockSend).toHaveBeenCalledTimes(1); }); }); @@ -529,6 +578,117 @@ describe("S3 download progress tracker transform body", () => { }); }); +// --- listTree(): the same listing, reported and interruptible while it runs --- +// +// Without it, listTreeForCollection() falls back to list(), which says nothing until it has +// finished and cannot be cancelled at all. Harmless while a listing stopped at 1000 keys, and +// not once it paginates: a large bucket lists for minutes behind a frozen progress row. +describe("S3 listTree()", () => { + // mockReset on top of clearAllMocks, because clearing only drops recorded calls: a + // `mockResolvedValueOnce` a test never consumed stays queued and answers the next one. + beforeEach(() => { + vi.clearAllMocks(); + mockSend.mockReset(); + }); + + const twoPages = () => { + mockSend + .mockResolvedValueOnce({ + Contents: [ + { Key: "Job/a.sql", Size: 10, LastModified: new Date() }, + { Key: "Job/b.sql", Size: 20, LastModified: new Date() }, + ], + IsTruncated: true, + NextContinuationToken: "page-2", + }) + .mockResolvedValueOnce({ + Contents: [{ Key: "Job/c.sql", Size: 30, LastModified: new Date() }], + IsTruncated: false, + }); + }; + + it("returns every page and reports no pruned directories", async () => { + twoPages(); + + const result = await S3GenericAdapter.listTree!(genericConfig, "Job"); + + expect(result.files.map((f) => f.name)).toEqual(["a.sql", "b.sql", "c.sql"]); + expect(result.pruned).toEqual([]); + expect(result.unsupportedSymlinks).toBeUndefined(); + }); + + it("reports progress once per page with a growing file count", async () => { + twoPages(); + const seen: number[] = []; + + await S3GenericAdapter.listTree!(genericConfig, "Job", { + onProgress: ({ files }) => seen.push(files), + }); + + expect(seen).toEqual([2, 3]); + }); + + it("reports a flat scan as having no directories", async () => { + mockSend.mockResolvedValue({ + Contents: [{ Key: "Job/a.sql", Size: 10, LastModified: new Date() }], + }); + const seen: Array<{ directories: number; prunedDirectories: number; currentPath: string }> = []; + + await S3GenericAdapter.listTree!(genericConfig, "Job", { + onProgress: ({ directories, prunedDirectories, currentPath }) => + seen.push({ directories, prunedDirectories, currentPath }), + }); + + expect(seen).toEqual([{ directories: 0, prunedDirectories: 0, currentPath: "" }]); + }); + + it("stops when the signal is already aborted, without listing anything", async () => { + const controller = new AbortController(); + controller.abort(); + + await expect( + S3GenericAdapter.listTree!(genericConfig, "Job", { signal: controller.signal }) + ).rejects.toThrow(); + expect(mockSend).not.toHaveBeenCalled(); + }); + + it("stops between pages when the signal fires mid-listing", async () => { + const controller = new AbortController(); + mockSend + .mockImplementationOnce(async () => { + controller.abort(); + return { + Contents: [{ Key: "Job/a.sql", Size: 10, LastModified: new Date() }], + IsTruncated: true, + NextContinuationToken: "page-2", + }; + }) + .mockResolvedValueOnce({ Contents: [], IsTruncated: false }); + + await expect( + S3GenericAdapter.listTree!(genericConfig, "Job", { signal: controller.signal }) + ).rejects.toThrow(); + // The second page is never requested. + expect(mockSend).toHaveBeenCalledTimes(1); + }); + + it("is exposed by every S3 adapter variant", () => { + for (const { adapter } of adapters) { + expect(typeof adapter.listTree).toBe("function"); + } + }); + + it("returns prefix-relative paths, same as list()", async () => { + mockSend.mockResolvedValue({ + Contents: [{ Key: "test/Images/a.jpg", Size: 10, LastModified: new Date() }], + }); + + const result = await S3GenericAdapter.listTree!({ ...genericConfig, pathPrefix: "test" }, ""); + + expect(result.files[0].path).toBe("Images/a.jpg"); + }); +}); + // --- pathPrefix: the adapter root, honoured symmetrically across every operation --- // // The regression this pins: list() used to return full bucket keys (prefix included) while @@ -557,6 +717,26 @@ describe("S3 pathPrefix is the adapter root", () => { expect(mockSend.mock.calls[0][0].Prefix).toBe("test/"); }); + // The prefix strip runs per object, so it has to survive pagination. A second page whose + // keys still carried the prefix would produce exactly the "test/test/..." drift above. + it("strips the prefix on every page of a paginated listing", async () => { + mockSend + .mockResolvedValueOnce({ + Contents: [{ Key: "test/Images/a.jpg", Size: 10, LastModified: new Date() }], + IsTruncated: true, + NextContinuationToken: "page-2", + }) + .mockResolvedValueOnce({ + Contents: [{ Key: "test/Java/b.jar", Size: 20, LastModified: new Date() }], + IsTruncated: false, + }); + + const result = await S3GenericAdapter.list(prefixed, ""); + + expect(result.map((f) => f.path)).toEqual(["Images/a.jpg", "Java/b.jar"]); + expect(mockSend.mock.calls[1][0].Prefix).toBe("test/"); + }); + it("download() re-applies the prefix to a prefix-relative path", async () => { const body = Readable.from(["x"]); (body as any).transformToString = vi.fn(); diff --git a/tests/unit/lib/core/retention.test.ts b/tests/unit/lib/core/retention.test.ts index b2198ffe..6845793f 100644 --- a/tests/unit/lib/core/retention.test.ts +++ b/tests/unit/lib/core/retention.test.ts @@ -1,5 +1,9 @@ import { describe, it, expect } from "vitest"; -import { DEFAULT_RETENTION_CONFIG } from "@/lib/core/retention"; +import { + DEFAULT_HOURLY_TIER, + DEFAULT_RETENTION_CONFIG, + RetentionConfigurationSchema, +} from "@/lib/core/retention"; describe("DEFAULT_RETENTION_CONFIG", () => { it("has mode NONE", () => { @@ -11,3 +15,60 @@ describe("DEFAULT_RETENTION_CONFIG", () => { expect(DEFAULT_RETENTION_CONFIG.smart).toBeUndefined(); }); }); + +describe("RetentionConfigurationSchema", () => { + const smart = { daily: 7, weekly: 4, monthly: 12, yearly: 2 }; + + it("accepts a smart policy without an hourly tier", () => { + const parsed = RetentionConfigurationSchema.parse({ mode: "SMART", smart }); + + expect(parsed.smart?.hourly).toBeUndefined(); + expect(parsed.smart?.daily).toBe(7); + }); + + it("accepts an hourly tier", () => { + const parsed = RetentionConfigurationSchema.parse({ + mode: "SMART", + smart: { ...smart, hourly: DEFAULT_HOURLY_TIER }, + }); + + expect(parsed.smart?.hourly).toBe(24); + }); + + it("coerces a numeric string tier, since form and API input arrives as text", () => { + const parsed = RetentionConfigurationSchema.parse({ + mode: "SMART", + smart: { ...smart, hourly: "24" }, + }); + + expect(parsed.smart?.hourly).toBe(24); + }); + + it("rejects a negative tier", () => { + expect(() => + RetentionConfigurationSchema.parse({ mode: "SMART", smart: { ...smart, hourly: -1 } }) + ).toThrow(); + }); + + it("rejects a fractional tier", () => { + expect(() => + RetentionConfigurationSchema.parse({ mode: "SMART", smart: { ...smart, hourly: 1.5 } }) + ).toThrow(); + }); + + it("rejects a non numeric tier", () => { + expect(() => + RetentionConfigurationSchema.parse({ mode: "SMART", smart: { ...smart, daily: "many" } }) + ).toThrow(); + }); + + it("rejects a keepCount below 1, which would delete every backup", () => { + expect(() => + RetentionConfigurationSchema.parse({ mode: "SIMPLE", simple: { keepCount: 0 } }) + ).toThrow(); + }); + + it("rejects an unknown mode", () => { + expect(() => RetentionConfigurationSchema.parse({ mode: "KEEP_LAST" })).toThrow(); + }); +}); diff --git a/tests/unit/lib/utils.test.ts b/tests/unit/lib/utils.test.ts index 1c8197f2..d94bc27c 100644 --- a/tests/unit/lib/utils.test.ts +++ b/tests/unit/lib/utils.test.ts @@ -5,6 +5,7 @@ import { formatDuration, formatTwoFactorCode, compareVersions, + isValidTimezone, } from "@/lib/utils"; describe("cn", () => { @@ -118,3 +119,44 @@ describe("compareVersions", () => { expect(compareVersions("beta", "beta")).toBe(0); }); }); + +describe("isValidTimezone", () => { + it("accepts UTC", () => { + expect(isValidTimezone("UTC")).toBe(true); + }); + + it("accepts ordinary IANA zones", () => { + expect(isValidTimezone("Europe/Zurich")).toBe(true); + expect(isValidTimezone("America/New_York")).toBe(true); + expect(isValidTimezone("America/Argentina/Buenos_Aires")).toBe(true); + }); + + // Node's Intl.supportedValuesOf('timeZone') only lists ICU's canonical IDs, which kept the + // legacy names. Browsers offering the primary name used to produce a value the server rejected. + it("accepts renamed zones the ICU canonical list omits", () => { + expect(isValidTimezone("Asia/Kolkata")).toBe(true); + expect(isValidTimezone("Europe/Kyiv")).toBe(true); + expect(isValidTimezone("Asia/Ho_Chi_Minh")).toBe(true); + expect(isValidTimezone("America/Nuuk")).toBe(true); + expect(isValidTimezone("Asia/Kathmandu")).toBe(true); + }); + + it("accepts the legacy name of a renamed zone", () => { + expect(isValidTimezone("Asia/Calcutta")).toBe(true); + expect(isValidTimezone("Europe/Kiev")).toBe(true); + }); + + it("rejects unknown zones", () => { + expect(isValidTimezone("Not/AZone")).toBe(false); + expect(isValidTimezone("Europe/Zurich ")).toBe(false); + }); + + it("rejects empty input", () => { + expect(isValidTimezone("")).toBe(false); + }); + + it("rejects bare UTC offsets, which carry no DST rules", () => { + expect(isValidTimezone("+05:30")).toBe(false); + expect(isValidTimezone("-08:00")).toBe(false); + }); +}); diff --git a/tests/unit/lint-guards/adapter-transport.test.ts b/tests/unit/lint-guards/adapter-transport.test.ts index ad572af5..3c75885f 100644 --- a/tests/unit/lint-guards/adapter-transport.test.ts +++ b/tests/unit/lint-guards/adapter-transport.test.ts @@ -124,6 +124,7 @@ const ADAPTER_RULES: Rule[] = [ pattern: /new\s+sql\.ConnectionPool/, allowed: { "database/mssql/pool.ts": "withPool() is the single wiring point that applies the SSH tunnel.", + "database/azure-sql/pool.ts": "Its own withPool(), forked rather than shared so a change to the MSSQL tunnelling cannot silently alter it, and so encryption stays pinned on.", }, reason: "Bypassing withPool() skips the tunnel and silently connects to the wrong machine.", }, @@ -316,6 +317,13 @@ describe("transport lint guards", () => { expect(violations, formatViolationReport(violations, rule)).toEqual([]); }); + // Timeout raised well above the default, because this is the one test here that + // imports the entire adapter registry - aws-sdk, googleapis, dropbox, ssh2, mssql + // and everything else behind it. On its own that takes under two seconds, but in + // the full suite it competes with 300-odd other files for CPU and occasionally + // crossed the 5 s default. The test asserts a structural property and has no + // business failing on machine load, and a limit that drifts closer to the edge + // with every adapter added is a trap for whoever adds the next one. it("wires a transport for every SSH-capable adapter", async () => { // Structural, not textual: an adapter that offers an SSH credential // slot but resolves to a direct host would accept SSH settings in the @@ -344,7 +352,7 @@ describe("transport lint guards", () => { `These adapters accept an SSH credential but resolve no transport:\n ${unwired.join("\n ")}\n` + "Add a `transport` resolver or a `connectionMode` field to the config schema.", ).toEqual([]); - }); + }, 30_000); it("passes a host to every adapter call in the integration tests", () => { // `test` and `ping` keep an optional host on BaseAdapter, because diff --git a/tests/unit/runner/steps/05-retention.test.ts b/tests/unit/runner/steps/05-retention.test.ts index 36b66f61..326ca885 100644 --- a/tests/unit/runner/steps/05-retention.test.ts +++ b/tests/unit/runner/steps/05-retention.test.ts @@ -270,6 +270,67 @@ describe('stepRetention', () => { await expect(stepRetention(ctx)).resolves.not.toThrow(); }); + it('warns by name when a file mtime disagrees with its recorded creation time', async () => { + // A destination whose modification times were reset has to be visible in the run + // log, otherwise the only symptom is backups quietly disappearing. + const { RetentionService } = await import('@/services/backup/retention-service'); + const file = { + name: 'moved.sql', + path: '/backups/moved.sql', + size: 1024, + lastModified: new Date('2026-06-08T12:00:00Z'), + }; + (RetentionService.calculateRetention as ReturnType).mockReturnValue({ keep: [file], delete: [], keptForChain: [] }); + + const dest = makeDestination({ + adapter: { + upload: vi.fn(), + list: vi.fn().mockResolvedValue([ + file, + { name: 'moved.sql.meta.json', path: '/backups/moved.sql.meta.json', size: 200, lastModified: new Date() }, + ]), + delete: vi.fn(), + read: vi.fn().mockResolvedValue(JSON.stringify({ timestamp: '2026-06-01T12:00:00Z' })), + } as any, + }); + const ctx = makeCtx({ destinations: [dest] }); + + await stepRetention(ctx); + + const warnings = (ctx.log as ReturnType).mock.calls.filter(c => c[1] === 'warning'); + expect(warnings.some(c => String(c[0]).includes('moved.sql'))).toBe(true); + expect(warnings.some(c => String(c[0]).includes('2026-06-01T12:00:00.000Z'))).toBe(true); + }); + + it('stays quiet when the recorded time and the mtime agree', async () => { + const { RetentionService } = await import('@/services/backup/retention-service'); + const file = { + name: 'normal.sql', + path: '/backups/normal.sql', + size: 1024, + lastModified: new Date('2026-06-08T12:00:30Z'), + }; + (RetentionService.calculateRetention as ReturnType).mockReturnValue({ keep: [file], delete: [], keptForChain: [] }); + + const dest = makeDestination({ + adapter: { + upload: vi.fn(), + list: vi.fn().mockResolvedValue([ + file, + { name: 'normal.sql.meta.json', path: '/backups/normal.sql.meta.json', size: 200, lastModified: new Date() }, + ]), + delete: vi.fn(), + read: vi.fn().mockResolvedValue(JSON.stringify({ timestamp: '2026-06-08T12:00:00Z' })), + } as any, + }); + const ctx = makeCtx({ destinations: [dest] }); + + await stepRetention(ctx); + + const warnings = (ctx.log as ReturnType).mock.calls.filter(c => c[1] === 'warning'); + expect(warnings).toHaveLength(0); + }); + it('triggers storage stats cache refresh when at least one file was deleted', async () => { const { RetentionService } = await import('@/services/backup/retention-service'); const { refreshStorageStatsCache } = await import('@/services/dashboard-service'); diff --git a/tests/unit/runner/steps/retention-sidecars.test.ts b/tests/unit/runner/steps/retention-sidecars.test.ts new file mode 100644 index 00000000..5aecfb61 --- /dev/null +++ b/tests/unit/runner/steps/retention-sidecars.test.ts @@ -0,0 +1,241 @@ +import { describe, it, expect, vi } from 'vitest'; +import { FileInfo, StorageAdapter } from '@/lib/core/interfaces'; +import { loadBackupSidecars, TIMESTAMP_DRIFT_WARNING_MS } from '@/lib/runner/steps/retention-sidecars'; + +const MTIME = new Date('2026-06-08T12:00:00Z'); + +const backup = (name: string, mtime: Date = MTIME): FileInfo => ({ + name, + path: `/job/${name}`, + size: 1024, + lastModified: mtime, +}); + +const sidecar = (name: string): FileInfo => ({ + name: `${name}.meta.json`, + path: `/job/${name}.meta.json`, + size: 200, + lastModified: MTIME, +}); + +/** + * A read() that records how many calls are in flight at once, which is the only way to + * tell a batched loader from a sequential one from the outside. + */ +function trackingAdapter( + contentFor: (remotePath: string) => string | null | Promise, + readConcurrency?: number +) { + const state = { inFlight: 0, peak: 0, calls: [] as string[] }; + const adapter = { + readConcurrency, + read: vi.fn(async (_config: unknown, remotePath: string) => { + state.calls.push(remotePath); + state.inFlight++; + state.peak = Math.max(state.peak, state.inFlight); + try { + // Yield so concurrent calls genuinely overlap rather than resolving inline. + await new Promise((resolve) => setTimeout(resolve, 1)); + return await contentFor(remotePath); + } finally { + state.inFlight--; + } + }), + } as unknown as StorageAdapter; + return { adapter, state }; +} + +describe('loadBackupSidecars', () => { + describe('concurrency', () => { + it('runs at most the declared number of reads at once', async () => { + const backups = Array.from({ length: 20 }, (_, i) => backup(`b${i}.sql`)); + const listing = [...backups, ...backups.map((b) => sidecar(b.name))]; + const { adapter, state } = trackingAdapter(() => '{}', 8); + + await loadBackupSidecars(adapter, {}, listing, backups); + + expect(state.calls).toHaveLength(20); + expect(state.peak).toBeLessThanOrEqual(8); + expect(state.peak).toBeGreaterThan(1); + }); + + it('stays sequential for an adapter that declares nothing', async () => { + const backups = Array.from({ length: 6 }, (_, i) => backup(`b${i}.sql`)); + const listing = [...backups, ...backups.map((b) => sidecar(b.name))]; + const { adapter, state } = trackingAdapter(() => '{}'); + + await loadBackupSidecars(adapter, {}, listing, backups); + + expect(state.calls).toHaveLength(6); + expect(state.peak).toBe(1); + }); + + it('treats a declared 0 as sequential rather than as no reads at all', async () => { + const backups = [backup('a.sql')]; + const listing = [...backups, sidecar('a.sql')]; + const { adapter, state } = trackingAdapter(() => '{}', 0); + + await loadBackupSidecars(adapter, {}, listing, backups); + + expect(state.calls).toHaveLength(1); + expect(state.peak).toBe(1); + }); + }); + + describe('skipping reads the listing already rules out', () => { + it('does not read a sidecar the listing does not contain', async () => { + const backups = [backup('has-meta.sql'), backup('no-meta.sql')]; + const listing = [...backups, sidecar('has-meta.sql')]; + const { adapter, state } = trackingAdapter(() => '{}', 8); + + await loadBackupSidecars(adapter, {}, listing, backups); + + expect(state.calls).toEqual(['/job/has-meta.sql.meta.json']); + }); + + it('falls back to trying every backup when the listing reports no sidecars at all', async () => { + // An adapter whose list() filters sidecars out must not silently lose lock and + // chain detection, so an empty sidecar set disables the optimisation. + const backups = [backup('a.sql'), backup('b.sql')]; + const { adapter, state } = trackingAdapter(() => '{}', 8); + + await loadBackupSidecars(adapter, {}, backups, backups); + + expect(state.calls).toHaveLength(2); + }); + + it('does nothing at all for an adapter without read()', async () => { + const backups = [backup('a.sql')]; + + const result = await loadBackupSidecars({} as StorageAdapter, {}, backups, backups); + + expect(result).toEqual({ withTimestamp: 0, drifted: [] }); + expect(backups[0].backupTimestamp).toBeUndefined(); + }); + }); + + describe('metadata applied to the file', () => { + it('sets locked and chainId from the sidecar', async () => { + const backups = [backup('a.sql'), backup('b.sql')]; + const listing = [...backups, sidecar('a.sql'), sidecar('b.sql')]; + const { adapter } = trackingAdapter((p) => + p.includes('a.sql') + ? JSON.stringify({ locked: true }) + : JSON.stringify({ chain: { id: 'chain-1' } }) + , 8); + + await loadBackupSidecars(adapter, {}, listing, backups); + + expect(backups[0].locked).toBe(true); + expect(backups[1].chainId).toBe('chain-1'); + }); + + it('takes a valid timestamp as the backup creation time', async () => { + const backups = [backup('a.sql')]; + const listing = [...backups, sidecar('a.sql')]; + const { adapter } = trackingAdapter(() => + JSON.stringify({ timestamp: '2026-06-08T11:30:00.000Z' }) + , 8); + + const result = await loadBackupSidecars(adapter, {}, listing, backups); + + expect(backups[0].backupTimestamp?.toISOString()).toBe('2026-06-08T11:30:00.000Z'); + expect(result.withTimestamp).toBe(1); + }); + + it('leaves the timestamp unset when the sidecar has none', async () => { + const backups = [backup('a.sql')]; + const listing = [...backups, sidecar('a.sql')]; + const { adapter } = trackingAdapter(() => JSON.stringify({ locked: false }), 8); + + const result = await loadBackupSidecars(adapter, {}, listing, backups); + + expect(backups[0].backupTimestamp).toBeUndefined(); + expect(result.withTimestamp).toBe(0); + }); + + it('rejects an unparsable timestamp instead of storing an Invalid Date', async () => { + // An Invalid Date poisons every comparison it takes part in and would sort + // unpredictably against real dates. + const backups = [backup('a.sql')]; + const listing = [...backups, sidecar('a.sql')]; + const { adapter } = trackingAdapter(() => JSON.stringify({ timestamp: 'not a date' }), 8); + + await loadBackupSidecars(adapter, {}, listing, backups); + + expect(backups[0].backupTimestamp).toBeUndefined(); + }); + }); + + describe('failures are contained', () => { + it('a throwing read leaves the other backups annotated', async () => { + const backups = [backup('bad.sql'), backup('good.sql')]; + const listing = [...backups, sidecar('bad.sql'), sidecar('good.sql')]; + const { adapter } = trackingAdapter((p) => { + if (p.includes('bad.sql')) throw new Error('network'); + return JSON.stringify({ timestamp: '2026-06-08T11:00:00.000Z' }); + }, 8); + + const result = await loadBackupSidecars(adapter, {}, listing, backups); + + expect(backups[0].backupTimestamp).toBeUndefined(); + expect(backups[1].backupTimestamp?.toISOString()).toBe('2026-06-08T11:00:00.000Z'); + expect(result.withTimestamp).toBe(1); + }); + + it('malformed JSON is treated as no sidecar', async () => { + const backups = [backup('a.sql')]; + const listing = [...backups, sidecar('a.sql')]; + const { adapter } = trackingAdapter(() => '{ not json', 8); + + const result = await loadBackupSidecars(adapter, {}, listing, backups); + + expect(result.withTimestamp).toBe(0); + expect(backups[0].backupTimestamp).toBeUndefined(); + }); + }); + + describe('drift reporting', () => { + it('reports a backup whose mtime disagrees with its recorded time', async () => { + const recorded = new Date(MTIME.getTime() - TIMESTAMP_DRIFT_WARNING_MS - 1000); + const backups = [backup('moved.sql')]; + const listing = [...backups, sidecar('moved.sql')]; + const { adapter } = trackingAdapter(() => + JSON.stringify({ timestamp: recorded.toISOString() }) + , 8); + + const result = await loadBackupSidecars(adapter, {}, listing, backups); + + expect(result.drifted).toHaveLength(1); + expect(result.drifted[0].file.name).toBe('moved.sql'); + expect(result.drifted[0].recorded.toISOString()).toBe(recorded.toISOString()); + expect(result.drifted[0].modified.toISOString()).toBe(MTIME.toISOString()); + }); + + it('stays quiet for the normal small gap between upload and mtime', async () => { + const recorded = new Date(MTIME.getTime() - 30_000); + const backups = [backup('normal.sql')]; + const listing = [...backups, sidecar('normal.sql')]; + const { adapter } = trackingAdapter(() => + JSON.stringify({ timestamp: recorded.toISOString() }) + , 8); + + const result = await loadBackupSidecars(adapter, {}, listing, backups); + + expect(result.drifted).toEqual([]); + expect(result.withTimestamp).toBe(1); + }); + }); + + it('matches sidecars for paths listed with backslashes', async () => { + // local.ts builds paths with path.relative, which yields backslashes on Windows. + const file: FileInfo = { name: 'a.sql', path: 'job\\a.sql', size: 10, lastModified: MTIME }; + const meta: FileInfo = { name: 'a.sql.meta.json', path: 'job\\a.sql.meta.json', size: 10, lastModified: MTIME }; + const { adapter, state } = trackingAdapter(() => JSON.stringify({ locked: true }), 8); + + await loadBackupSidecars(adapter, {}, [file, meta], [file]); + + expect(state.calls).toHaveLength(1); + expect(file.locked).toBe(true); + }); +}); diff --git a/tests/unit/services/retention-service.test.ts b/tests/unit/services/retention-service.test.ts index 38cf2130..c7c6ff9b 100644 --- a/tests/unit/services/retention-service.test.ts +++ b/tests/unit/services/retention-service.test.ts @@ -2,7 +2,7 @@ import { describe, it, expect } from 'vitest'; import { RetentionService } from '@/services/backup/retention-service'; import { FileInfo } from '@/lib/core/interfaces'; import { RetentionConfiguration } from '@/lib/core/retention'; -import { subDays, subWeeks, subMonths, subYears } from 'date-fns'; +import { subDays, subWeeks, subMonths, subYears, subHours } from 'date-fns'; // Helper to generate mock files const createMockFiles = (dates: Date[]): FileInfo[] => { @@ -651,6 +651,323 @@ describe('RetentionService', () => { }); }); +// โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +// Hourly tier +// +// Fixed reference: Monday 2026-06-08 12:00 UTC. All buckets are evaluated in UTC +// so an hour bucket is unambiguous. +// โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +describe('RetentionService - hourly tier', () => { + const REF = new Date('2026-06-08T12:00:00Z'); + + it('hourly=24: keeps the newest 24 unique hours out of 30 hourly backups', () => { + const files = createMockFiles( + Array.from({ length: 30 }, (_, i) => subHours(REF, i)) + ); + const policy: RetentionConfiguration = { + mode: 'SMART', + smart: { hourly: 24, daily: 0, weekly: 0, monthly: 0, yearly: 0 } + }; + + const result = RetentionService.calculateRetention(files, policy, 'UTC'); + + expect(result.keep).toHaveLength(24); + expect(result.delete).toHaveLength(6); + + const keptTimes = result.keep.map(f => f.lastModified.getTime()); + for (let i = 0; i < 24; i++) { + expect(keptTimes).toContain(subHours(REF, i).getTime()); + } + // Hours 24 through 29 fall outside the tier. + const deletedTimes = result.delete.map(f => f.lastModified.getTime()); + for (let i = 24; i < 30; i++) { + expect(deletedTimes).toContain(subHours(REF, i).getTime()); + } + }); + + it('keeps only the newest backup of an hour that holds several', () => { + const files = createMockFiles([ + new Date('2026-06-08T10:45:00Z'), // hour 10, newest + new Date('2026-06-08T10:20:00Z'), // hour 10 + new Date('2026-06-08T10:05:00Z'), // hour 10 + new Date('2026-06-08T09:30:00Z'), // hour 09 + ]); + const policy: RetentionConfiguration = { + mode: 'SMART', + smart: { hourly: 2, daily: 0, weekly: 0, monthly: 0, yearly: 0 } + }; + + const result = RetentionService.calculateRetention(files, policy, 'UTC'); + + expect(result.keep.map(f => f.lastModified.toISOString()).sort()).toEqual([ + '2026-06-08T09:30:00.000Z', + '2026-06-08T10:45:00.000Z', + ]); + expect(result.delete).toHaveLength(2); + }); + + it('daily adds days on top of what hourly covers rather than overlapping it', () => { + // hourly=3 takes the three newest hours, all on Jun 8. daily=2 then adds Jun 7 + // and Jun 6, because Jun 8 is already covered. Jun 5 falls out. + const files = createMockFiles([ + new Date('2026-06-08T12:00:00Z'), + new Date('2026-06-08T11:00:00Z'), + new Date('2026-06-08T10:00:00Z'), + new Date('2026-06-07T12:00:00Z'), + new Date('2026-06-06T12:00:00Z'), + new Date('2026-06-05T12:00:00Z'), + ]); + const policy: RetentionConfiguration = { + mode: 'SMART', + smart: { hourly: 3, daily: 2, weekly: 0, monthly: 0, yearly: 0 } + }; + + const result = RetentionService.calculateRetention(files, policy, 'UTC'); + + expect(result.keep).toHaveLength(5); + expect(result.delete.map(f => f.lastModified.toISOString())).toEqual([ + '2026-06-05T12:00:00.000Z', + ]); + }); + + it('a policy stored before the tier existed deletes exactly as it did before', () => { + // The regression that matters. `undefined <= 0` is false in JavaScript, so a bare + // comparison would let the hourly tier run with no limit and keep one backup per + // hour forever, silently turning off deletion for every policy already in the wild. + const files = createMockFiles([ + new Date('2026-06-08T12:00:00Z'), + new Date('2026-06-08T11:00:00Z'), + new Date('2026-06-07T12:00:00Z'), + new Date('2026-06-07T11:00:00Z'), + new Date('2026-06-06T12:00:00Z'), + new Date('2026-06-06T11:00:00Z'), + ]); + const policy: RetentionConfiguration = { + mode: 'SMART', + smart: { daily: 2, weekly: 0, monthly: 0, yearly: 0 } + }; + + const result = RetentionService.calculateRetention(files, policy, 'UTC'); + + expect(result.keep.map(f => f.lastModified.toISOString()).sort()).toEqual([ + '2026-06-07T12:00:00.000Z', + '2026-06-08T12:00:00.000Z', + ]); + expect(result.delete).toHaveLength(4); + }); + + it('hourly=0 behaves identically to a policy without the field', () => { + const dates = [ + new Date('2026-06-08T12:00:00Z'), + new Date('2026-06-08T11:00:00Z'), + new Date('2026-06-07T12:00:00Z'), + new Date('2026-06-06T12:00:00Z'), + ]; + + const withZero = RetentionService.calculateRetention( + createMockFiles(dates), + { mode: 'SMART', smart: { hourly: 0, daily: 2, weekly: 0, monthly: 0, yearly: 0 } }, + 'UTC' + ); + const withoutField = RetentionService.calculateRetention( + createMockFiles(dates), + { mode: 'SMART', smart: { daily: 2, weekly: 0, monthly: 0, yearly: 0 } }, + 'UTC' + ); + + expect(withZero.keep.map(f => f.name).sort()).toEqual(withoutField.keep.map(f => f.name).sort()); + expect(withZero.delete.map(f => f.name).sort()).toEqual(withoutField.delete.map(f => f.name).sort()); + }); + + it('locked backups survive the hourly tier without consuming a slot', () => { + const files = createMockFiles([ + new Date('2026-06-08T12:00:00Z'), + new Date('2026-06-08T11:00:00Z'), + new Date('2026-06-08T10:00:00Z'), + new Date('2026-06-08T09:00:00Z'), + ]); + files[3].locked = true; // the oldest, well past the tier + + const policy: RetentionConfiguration = { + mode: 'SMART', + smart: { hourly: 2, daily: 0, weekly: 0, monthly: 0, yearly: 0 } + }; + + const result = RetentionService.calculateRetention(files, policy, 'UTC'); + + // 2 hourly slots plus the locked file, which is not counted against them. + expect(result.keep.map(f => f.lastModified.toISOString()).sort()).toEqual([ + '2026-06-08T09:00:00.000Z', + '2026-06-08T11:00:00.000Z', + '2026-06-08T12:00:00.000Z', + ]); + expect(result.delete.map(f => f.lastModified.toISOString())).toEqual([ + '2026-06-08T10:00:00.000Z', + ]); + }); + + it('holds a whole incremental chain back when an hourly slot keeps one of its snapshots', () => { + const files = createMockFiles([ + new Date('2026-06-08T12:00:00Z'), + new Date('2026-06-08T11:00:00Z'), + new Date('2026-06-08T10:00:00Z'), + ]); + files[1].chainId = 'chain-a'; + files[2].chainId = 'chain-a'; + + const policy: RetentionConfiguration = { + mode: 'SMART', + smart: { hourly: 2, daily: 0, weekly: 0, monthly: 0, yearly: 0 } + }; + + const result = RetentionService.calculateRetention(files, policy, 'UTC'); + + expect(result.delete).toHaveLength(0); + expect(result.keptForChain.map(f => f.lastModified.toISOString())).toEqual([ + '2026-06-08T10:00:00.000Z', + ]); + }); +}); + +// โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +// Time source +// +// Retention buckets by the time DBackup recorded when it wrote the backup, and only +// falls back to the destination's modification time when there is none. +// โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +describe('RetentionService - time source', () => { + const withTimes = (entries: { name: string; mtime: string; recorded?: string }[]): FileInfo[] => + entries.map((e) => ({ + name: e.name, + path: `/job/${e.name}`, + size: 1024, + lastModified: new Date(e.mtime), + ...(e.recorded ? { backupTimestamp: new Date(e.recorded) } : {}), + })); + + it('buckets by the recorded time, not by the modification time', () => { + // Both files carry today's mtime, but were written on different days. Daily=2 has + // to see two days, which is only true if it reads the recorded time. + const files = withTimes([ + { name: 'a.sql', mtime: '2026-06-08T12:00:00Z', recorded: '2026-06-08T12:00:00Z' }, + { name: 'b.sql', mtime: '2026-06-08T12:00:00Z', recorded: '2026-06-07T12:00:00Z' }, + ]); + + const result = RetentionService.calculateRetention( + files, + { mode: 'SMART', smart: { daily: 2, weekly: 0, monthly: 0, yearly: 0 } }, + 'UTC' + ); + + expect(result.keep).toHaveLength(2); + expect(result.delete).toHaveLength(0); + }); + + it('sorts by the recorded time, so the newest of a bucket is the one written last', () => { + // The mtimes claim b.sql is newer. The recorded times say otherwise, and the + // representative of the shared day has to be a.sql. + const files = withTimes([ + { name: 'a.sql', mtime: '2026-06-08T08:00:00Z', recorded: '2026-06-08T18:00:00Z' }, + { name: 'b.sql', mtime: '2026-06-08T20:00:00Z', recorded: '2026-06-08T09:00:00Z' }, + ]); + + const result = RetentionService.calculateRetention( + files, + { mode: 'SMART', smart: { daily: 1, weekly: 0, monthly: 0, yearly: 0 } }, + 'UTC' + ); + + expect(result.keep.map((f) => f.name)).toEqual(['a.sql']); + expect(result.delete.map((f) => f.name)).toEqual(['b.sql']); + }); + + it('falls back to the modification time for a backup without a recorded one', () => { + const files = withTimes([ + { name: 'new.sql', mtime: '2026-06-08T12:00:00Z' }, + { name: 'old.sql', mtime: '2026-06-01T12:00:00Z' }, + ]); + + const result = RetentionService.calculateRetention( + files, + { mode: 'SMART', smart: { daily: 1, weekly: 0, monthly: 0, yearly: 0 } }, + 'UTC' + ); + + expect(result.keep.map((f) => f.name)).toEqual(['new.sql']); + expect(result.delete.map((f) => f.name)).toEqual(['old.sql']); + }); + + it('sorts files with and without a recorded time into one order', () => { + const files = withTimes([ + { name: 'mtime-only.sql', mtime: '2026-06-07T12:00:00Z' }, + { name: 'recorded.sql', mtime: '2026-01-01T00:00:00Z', recorded: '2026-06-08T12:00:00Z' }, + { name: 'older.sql', mtime: '2026-06-06T12:00:00Z' }, + ]); + + const result = RetentionService.calculateRetention( + files, + { mode: 'SIMPLE', simple: { keepCount: 2 } }, + 'UTC' + ); + + expect(result.keep.map((f) => f.name).sort()).toEqual(['mtime-only.sql', 'recorded.sql']); + expect(result.delete.map((f) => f.name)).toEqual(['older.sql']); + }); + + it('survives a destination whose modification times were all reset', () => { + // The regression this exists for. A copy without -p, a migration, or a restore of + // the backup directory stamps every file with the same mtime. Judged by mtime the + // whole history collapses into one bucket and a single representative survives. + const reset = '2026-06-08T12:00:00Z'; + const files = withTimes([ + { name: 'd0.sql', mtime: reset, recorded: '2026-06-08T02:00:00Z' }, + { name: 'd1.sql', mtime: reset, recorded: '2026-06-07T02:00:00Z' }, + { name: 'd2.sql', mtime: reset, recorded: '2026-06-06T02:00:00Z' }, + { name: 'd3.sql', mtime: reset, recorded: '2026-06-05T02:00:00Z' }, + ]); + + const result = RetentionService.calculateRetention( + files, + { mode: 'SMART', smart: { daily: 3, weekly: 0, monthly: 0, yearly: 0 } }, + 'UTC' + ); + + expect(result.keep.map((f) => f.name).sort()).toEqual(['d0.sql', 'd1.sql', 'd2.sql']); + expect(result.delete.map((f) => f.name)).toEqual(['d3.sql']); + }); +}); + +describe('RetentionService - unreadable policy', () => { + const files = createMockFiles([ + new Date('2026-06-08T12:00:00Z'), + new Date('2026-06-07T12:00:00Z'), + new Date('2026-06-06T12:00:00Z'), + ]); + + // Nothing marks a file as kept when the mode carries no usable settings, and falling + // through to the delete list would wipe the destination. Keeping is the safe default. + it('keeps everything when SMART carries no smart settings', () => { + const result = RetentionService.calculateRetention(files, { mode: 'SMART' }); + + expect(result.keep).toHaveLength(3); + expect(result.delete).toHaveLength(0); + }); + + it('keeps everything when SIMPLE carries no simple settings', () => { + const result = RetentionService.calculateRetention(files, { mode: 'SIMPLE' }); + + expect(result.keep).toHaveLength(3); + expect(result.delete).toHaveLength(0); + }); + + it('keeps everything for a mode it does not recognise', () => { + const result = RetentionService.calculateRetention(files, { mode: 'KEEP_LAST' } as unknown as RetentionConfiguration); + + expect(result.keep).toHaveLength(3); + expect(result.delete).toHaveLength(0); + }); +}); + describe('RetentionService - incremental chains', () => { const at = (day: number) => new Date(2026, 0, day); const file = (name: string, day: number, chainId?: string, locked = false) => ({ diff --git a/tests/unit/services/templates/retention-policy-service.test.ts b/tests/unit/services/templates/retention-policy-service.test.ts index f12b8fad..99c96aef 100644 --- a/tests/unit/services/templates/retention-policy-service.test.ts +++ b/tests/unit/services/templates/retention-policy-service.test.ts @@ -22,7 +22,7 @@ import { deleteRetentionPolicy, parseRetentionPolicyConfig, } from "@/services/templates/retention-policy-service"; -import { NotFoundError, ServiceError } from "@/lib/logging/errors"; +import { NotFoundError, ServiceError, ValidationError } from "@/lib/logging/errors"; import type { RetentionConfiguration } from "@/lib/core/retention"; const makePolicy = (overrides: object = {}) => ({ @@ -113,6 +113,34 @@ describe("RetentionPolicyService", () => { ).rejects.toBeInstanceOf(ServiceError); }); + it("stores an hourly tier", async () => { + prismaMock.retentionPolicy.findUnique.mockResolvedValue(null); + prismaMock.retentionPolicy.create.mockResolvedValue(makePolicy() as any); + + await createRetentionPolicy({ + name: "Hourly GFS", + config: { mode: "SMART", smart: { hourly: 24, daily: 7, weekly: 4, monthly: 12, yearly: 2 } }, + }); + + const stored = JSON.parse( + prismaMock.retentionPolicy.create.mock.calls[0][0].data.config as string + ); + expect(stored.smart.hourly).toBe(24); + }); + + it("rejects a config with a negative tier instead of storing it", async () => { + prismaMock.retentionPolicy.findUnique.mockResolvedValue(null); + + await expect( + createRetentionPolicy({ + name: "Broken", + config: { mode: "SMART", smart: { hourly: -5, daily: 7, weekly: 4, monthly: 12, yearly: 2 } }, + }) + ).rejects.toBeInstanceOf(ValidationError); + + expect(prismaMock.retentionPolicy.create).not.toHaveBeenCalled(); + }); + it("clears previous default when isDefault is true", async () => { prismaMock.retentionPolicy.findUnique.mockResolvedValue(null); prismaMock.retentionPolicy.updateMany.mockResolvedValue({ count: 1 }); diff --git a/website/src/lib/adapter-icons.ts b/website/src/lib/adapter-icons.ts index cd3032fe..935b6d14 100644 --- a/website/src/lib/adapter-icons.ts +++ b/website/src/lib/adapter-icons.ts @@ -17,6 +17,7 @@ import slackIcon from "@iconify-icons/logos/slack-icon"; import teamsIcon from "@iconify-icons/logos/microsoft-teams"; import telegramIcon from "@iconify-icons/logos/telegram"; import dockerIcon from "@iconify-icons/logos/docker-icon"; +import azureIcon from "@iconify-icons/logos/azure"; // Simple Icons (monochrome - brand color applied via getAdapterColor) import mssqlIcon from "@iconify-icons/simple-icons/microsoftsqlserver"; @@ -63,6 +64,7 @@ const ADAPTER_ICON_MAP: Record = { redis: redisIcon, valkey: valkeyIcon, mssql: mssqlIcon, + "azure-sql": azureIcon, firebird: firebirdIcon, // Storage "local-filesystem": harddiskIcon, diff --git a/website/src/lib/content.ts b/website/src/lib/content.ts index 7df79dec..1c2db995 100644 --- a/website/src/lib/content.ts +++ b/website/src/lib/content.ts @@ -81,6 +81,7 @@ export const DATABASES: AdapterItem[] = [ { id: "redis", label: "Redis" }, { id: "valkey", label: "Valkey" }, { id: "mssql", label: "Microsoft SQL Server" }, + { id: "azure-sql", label: "Azure SQL Database (Beta)" }, { id: "firebird", label: "Firebird (Beta)" }, ]; @@ -138,7 +139,7 @@ export const FAQS = [ { question: "Which databases are supported?", answer: - "MySQL, MariaDB, PostgreSQL, MongoDB, SQLite, Redis, Valkey, Microsoft SQL Server, and Firebird (beta), with more engines added regularly.", + "MySQL, MariaDB, PostgreSQL, MongoDB, SQLite, Redis, Valkey, Microsoft SQL Server, Azure SQL Database (beta), and Firebird (beta), with more engines added regularly.", }, { question: "Can DBackup back up files and folders, not just databases?",