diff --git a/Dockerfile b/Dockerfile index 424990eb5..93196128d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,5 +43,7 @@ EXPOSE 10000 EXPOSE 10001 # Table Storage Port EXPOSE 10002 +# DFS (ADLS Gen2) Port +EXPOSE 10004 -CMD ["azurite", "-l", "/data", "--blobHost", "0.0.0.0","--queueHost", "0.0.0.0", "--tableHost", "0.0.0.0"] +CMD ["azurite", "-l", "/data", "--blobHost", "0.0.0.0", "--dfsHost", "0.0.0.0", "--queueHost", "0.0.0.0", "--tableHost", "0.0.0.0"] diff --git a/Dockerfile.Windows b/Dockerfile.Windows index 1d3250e63..f1298f1c9 100644 --- a/Dockerfile.Windows +++ b/Dockerfile.Windows @@ -67,9 +67,11 @@ EXPOSE 10000 EXPOSE 10001 # Table Storage Port EXPOSE 10002 +# DFS (ADLS Gen2) Port +EXPOSE 10004 ENTRYPOINT "cmd.exe /S /C" WORKDIR C:\\Node\\node-v22.12.0-win-x64\\ -CMD azurite -l c:/data --blobHost 0.0.0.0 --queueHost 0.0.0.0 --tableHost 0.0.0.0 \ No newline at end of file +CMD azurite -l c:/data --blobHost 0.0.0.0 --dfsHost 0.0.0.0 --queueHost 0.0.0.0 --tableHost 0.0.0.0 \ No newline at end of file diff --git a/README.md b/README.md index 79088a980..eb7b45be1 100644 --- a/README.md +++ b/README.md @@ -186,6 +186,8 @@ Following extension configurations are supported: - `azurite.blobHost` Blob service listening endpoint, by default 127.0.0.1 - `azurite.blobPort` Blob service listening port, by default 10000 +- `azurite.dfsHost` DFS service listening endpoint, by default 127.0.0.1 +- `azurite.dfsPort` DFS service listening port, by default 10004 - `azurite.blobKeepAliveTimeout` Blob service keep alive timeout in seconds, by default 5 - `azurite.queueHost` Queue service listening endpoint, by default 127.0.0.1 - `azurite.queuePort` Queue service listening port, by default 10001 @@ -214,17 +216,18 @@ Following extension configurations are supported: > Note. Find more docker images tags in ```bash -docker run -p 10000:10000 -p 10001:10001 -p 10002:10002 mcr.microsoft.com/azure-storage/azurite +docker run -p 10000:10000 -p 10004:10004 -p 10001:10001 -p 10002:10002 mcr.microsoft.com/azure-storage/azurite ``` `-p 10000:10000` will expose blob service's default listening port. +`-p 10004:10004` will expose dfs service's default listening port. `-p 10001:10001` will expose queue service's default listening port. `-p 10002:10002` will expose table service's default listening port. Or just run blob service: ```bash -docker run -p 10000:10000 mcr.microsoft.com/azure-storage/azurite azurite-blob --blobHost 0.0.0.0 +docker run -p 10000:10000 -p 10004:10004 mcr.microsoft.com/azure-storage/azurite azurite-blob --blobHost 0.0.0.0 --dfsHost 0.0.0.0 ``` #### Run Azurite V3 docker image with customized persisted data location @@ -317,6 +320,7 @@ You can customize the listening address per your requirements. ```cmd --blobHost 127.0.0.1 +--dfsHost 127.0.0.1 --queueHost 127.0.0.1 --tableHost 127.0.0.1 ``` @@ -325,13 +329,14 @@ You can customize the listening address per your requirements. ```cmd --blobHost 0.0.0.0 +--dfsHost 0.0.0.0 --queueHost 0.0.0.0 --tableHost 0.0.0.0 ``` ### Listening Port Configuration -Optional. By default, Azurite V3 will listen to 10000 as blob service port, and 10001 as queue service port, and 10002 as the table service port. +Optional. By default, Azurite V3 will listen to 10000 as blob service port, 10004 as dfs service port, 10001 as queue service port, and 10002 as the table service port. You can customize the listening port per your requirements. > Warning: After using a customized port, you need to update connection string or configurations correspondingly in your Storage Tools or SDKs. @@ -341,6 +346,7 @@ You can customize the listening port per your requirements. ```cmd --blobPort 8888 +--dfsPort 8889 --queuePort 9999 --tablePort 11111 ``` @@ -349,6 +355,7 @@ You can customize the listening port per your requirements. ```cmd --blobPort 0 +--dfsPort 0 --queuePort 0 --tablePort 0 ``` diff --git a/docs/designs/ADLS-gen2-parity.md b/docs/designs/ADLS-gen2-parity.md new file mode 100644 index 000000000..4f26078f0 --- /dev/null +++ b/docs/designs/ADLS-gen2-parity.md @@ -0,0 +1,187 @@ +# ADLS Gen2 Parity Implementation Plan + +## Context + +Azurite currently has a **thin DFS proxy layer** (port 10004) that translates a small subset of ADLS Gen2 DFS REST API calls to Blob REST API calls via HTTP proxying (axios). This covers only filesystem (container) create/delete/HEAD and account listing. Full ADLS Gen2 parity requires native support for path (file/directory) operations, the append-then-flush write pattern, rename/move, ACLs, and list paths — none of which can be achieved by simple query-parameter rewriting. + +## Architectural Decision: Hybrid (Native DFS Handlers + Shared Stores) + +Replace the HTTP proxy with a **native Express pipeline** in the DFS server that directly accesses `IBlobMetadataStore` and `IExtentStore` — the same store instances used by the blob server. + +``` +Port 10000 (Blob API) → Blob Handlers → IBlobMetadataStore + IExtentStore +Port 10004 (DFS API) → DFS Handlers → same IBlobMetadataStore + IExtentStore +``` + +**Why not keep proxying?** DFS operations like List Paths, Create Directory, Rename, ACLs, and append-then-flush have no single blob API equivalent. Proxying would require multi-call orchestration, lose atomicity, and add latency. + +### Directory Model + +Directories stored as **zero-length BlockBlobs with `hdi_isfolder=true` metadata** — matching Azure's real internal behavior. No separate table needed. + +### ACL Storage + +New fields on `BlobModel`: `dfsAclOwner`, `dfsAclGroup`, `dfsAclPermissions`, `dfsAcl`. LokiJS is schemaless (just add fields); SQL needs ALTER TABLE. + +--- + +## Phase 0: Foundation — Shared Store Access & HNS Flag + +**Goal:** Wire DFS server to share stores with blob server; enable HNS mode. + +| File | Change | +|------|--------| +| `src/blob/utils/constants.ts` | Set `EMULATOR_ACCOUNT_ISHIERARCHICALNAMESPACEENABLED = true` (or make configurable) | +| `src/blob/DfsProxyServer.ts` → rename to `DfsServer.ts` | Accept `IBlobMetadataStore` + `IExtentStore` in constructor | +| `src/blob/DfsProxyConfiguration.ts` → rename to `DfsConfiguration.ts` | Remove upstream host/port fields (no longer proxying) | +| `src/blob/BlobServer.ts` | Expose `metadataStore` and `extentStore` via public getters | +| `src/azurite.ts` | Pass shared stores to both BlobServer and DfsServer | +| `src/blob/main.ts` | Same wiring for standalone blob+dfs mode | +| `src/blob/DfsRequestListenerFactory.ts` | Rewrite: replace axios proxy with native Express pipeline + DFS routing | +| `src/blob/IBlobEnvironment.ts`, `BlobEnvironment.ts`, `src/common/Environment.ts`, `VSCEnvironment.ts` | Add `--enableHierarchicalNamespace` option | + +**Deliverable:** DFS server starts, shares data with blob, existing filesystem tests pass via direct store access. + +--- + +## Phase 1: Path CRUD + List Paths + +**Goal:** Create/delete/read files and directories, list paths — the core operations most ADLS Gen2 SDKs depend on. + +### New files to create + +| File | Purpose | +|------|---------| +| `src/blob/dfs/DfsContext.ts` | DFS request context (account, filesystem, path) — analogous to `BlobStorageContext` | +| `src/blob/dfs/DfsOperation.ts` | Enum of DFS operations for dispatch | +| `src/blob/dfs/DfsDispatchMiddleware.ts` | Routes requests by `resource` param, `action` param, method, and headers | +| `src/blob/dfs/DfsErrorFactory.ts` | JSON error responses (`PathNotFound`, `DirectoryNotEmpty`, etc.) | +| `src/blob/dfs/DfsSerializer.ts` | JSON response serialization (DFS uses JSON, not XML) | +| `src/blob/dfs/handlers/FilesystemHandler.ts` | Filesystem ops → container store operations | +| `src/blob/dfs/handlers/PathHandler.ts` | Path create/delete/read/getProperties + listPaths | + +### Operations implemented + +- **Create Path** (`PUT ?resource=file|directory`): Creates zero-length BlockBlob; directories get `hdi_isfolder=true` metadata; auto-creates intermediate directories +- **Delete Path** (`DELETE`): Files → `deleteBlob()`; directories with `recursive=true` → delete all blobs with prefix; `recursive=false` → 409 if non-empty +- **Get Path Properties** (`HEAD`): Returns `x-ms-resource-type: file|directory` header +- **Read Path** (`GET`): Streams file content via `downloadBlob()` (follows `BlobHandler.download()` pattern) +- **List Paths** (`GET ?resource=filesystem&directory=...&recursive=true|false`): JSON response with `paths` array; uses `listBlobs()` with prefix/delimiter; supports continuation via `x-ms-continuation` + +### Existing files modified + +| File | Change | +|------|--------| +| `src/blob/persistence/IBlobMetadataStore.ts` | Add `dfsResourceType`, ACL fields to `BlobModel` / `IBlobAdditionalProperties` | +| `src/blob/persistence/LokiBlobMetadataStore.ts` | No schema changes needed (schemaless) | +| `src/blob/persistence/SqlBlobMetadataStore.ts` | Add columns: `dfsResourceType`, `dfsAclOwner`, `dfsAclGroup`, `dfsAclPermissions`, `dfsAcl` | + +### Tests + +Extend `tests/blob/dfsProxy.test.ts`: +- Create file / directory, verify as blob +- Delete file / empty dir / non-empty dir with recursive +- Get properties with `x-ms-resource-type` +- Read file content +- List paths recursive and non-recursive +- Cross-API: create via DFS → read via Blob API and vice versa + +--- + +## Phase 2: Append-Flush Write Pattern + +**Goal:** Implement the DFS file write model (create empty → append chunks → flush to commit). + +### Key insight + +DFS append-then-flush maps directly to existing **BlockBlob uncommitted blocks** infrastructure: each `action=append` becomes a `stageBlock()`, and `action=flush` becomes `commitBlockList()`. No new persistence methods needed. + +### Changes to `src/blob/dfs/handlers/PathHandler.ts` + +- **`updatePath_Append(position, body)`**: Write body to `IExtentStore` as extent chunk; record as uncommitted block via `metadataStore.stageBlock()`; validate `position` matches current append offset; return 202 +- **`updatePath_Flush(position, close)`**: Commit all staged blocks via `metadataStore.commitBlockList()`; update content length to `position`; return 200 with updated ETag + +### Tests + +- Create → append 3 chunks → flush → read back, verify content +- Append with wrong position → 400 +- Large file (multi-MB) append + +--- + +## Phase 3: Rename/Move Path + +**Goal:** Atomic rename for files and directories. + +### New persistence methods + +| Method | Description | +|--------|-------------| +| `IBlobMetadataStore.renameBlob(src, dest)` | Atomic rename of single blob (metadata-only, no extent copy) | +| `IBlobMetadataStore.renameBlobsByPrefix(srcPrefix, destPrefix)` | Atomic rename of all blobs matching prefix (for directory rename) | + +### PathHandler addition + +- **`renamePath(x-ms-rename-source)`**: Parse source header → for files: `renameBlob()`; for directories: `renameBlobsByPrefix()`. Supports cross-filesystem rename and conditional headers. + +### Persistence implementations + +- **LokiJS**: Update document `containerName` and `name` properties +- **SQL**: `UPDATE ... SET name = REPLACE(name, oldPrefix, newPrefix) WHERE name LIKE 'prefix%'` in transaction + +### Tests + +- Rename file within filesystem / across filesystems +- Rename directory (verify children moved) +- Rename non-existent → 404 +- Rename with conditional headers + +--- + +## Phase 4: ACL Operations + +**Goal:** POSIX ACL get/set for emulator parity. + +### PathHandler additions + +- **`getAccessControl()`**: Read ACL fields from blob record → return as `x-ms-owner`, `x-ms-group`, `x-ms-permissions`, `x-ms-acl` headers. Defaults: `$superuser`/`$superuser`/`rwxr-x---` +- **`setAccessControl(owner, group, permissions, acl)`**: Validate ACL format → update blob record +- **`setAccessControlRecursive(mode, acl)`**: `mode` = set|modify|remove; iterate blobs under prefix; support continuation; return JSON with `directoriesSuccessful`, `filesSuccessful`, `failureCount` + +### Tests + +- Set/get ACL on file and directory +- Recursive ACL set on directory tree +- Default ACL values on new paths + +--- + +## Phase 5: Polish & Remaining Operations + +- **Set Filesystem Properties** (`PATCH ?resource=filesystem`) → `setContainerMetadata()` +- **`x-ms-properties` encoding/decoding** — new `src/blob/dfs/DfsPropertyEncoding.ts` utility (base64 key=value pairs) +- **DFS JSON error format**: `{"error":{"code":"...","message":"..."}}` +- **Lease support** on DFS paths (reuse blob lease infrastructure) +- **SAS validation** on DFS endpoints (reuse existing authenticators) +- **Content-MD5/CRC64 validation** on append + +--- + +## Verification Plan + +1. **Unit tests**: Extend `tests/blob/dfsProxy.test.ts` per phase +2. **Cross-API tests**: Verify DFS-created data is visible via Blob API and vice versa +3. **SDK integration**: Test with `@azure/storage-file-datalake` Node.js SDK against the emulator +4. **Manual smoke test**: Run Azurite, use Azure Storage Explorer with DFS endpoint +5. **Existing blob tests**: Ensure `npm test` still passes (no regression) + +--- + +## Critical Reference Files + +- `src/blob/handlers/ContainerHandler.ts` — pattern for handler ↔ store interaction +- `src/blob/handlers/BlockBlobHandler.ts` — `stageBlock`/`commitBlockList` for append-flush reuse +- `src/blob/handlers/BlobHandler.ts` — `download()` pattern for Read Path +- `src/blob/persistence/IBlobMetadataStore.ts` — store interface to extend +- `src/blob/generated/handlers/` — handler interface patterns +- `src/blob/middlewares/blobStorageContext.middleware.ts` — context extraction pattern for DfsContext diff --git a/package-lock.json b/package-lock.json index c79dba128..55b5266a2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -44,6 +44,7 @@ "@azure/core-rest-pipeline": "^1.2.0", "@azure/data-tables": "^13.0.1", "@azure/storage-blob": "^12.9.0", + "@azure/storage-file-datalake": "^12.29.0", "@azure/storage-queue": "^12.8.0", "@types/args": "^5.0.0", "@types/async": "^3.0.1", @@ -143,20 +144,77 @@ } }, "node_modules/@azure/core-client": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/@azure/core-client/-/core-client-1.5.0.tgz", - "integrity": "sha512-YNk8i9LT6YcFdFO+RRU0E4Ef+A8Y5lhXo6lz61rwbG8Uo7kSqh0YqK04OexiilM43xd6n3Y9yBhLnb1NFNI9dA==", + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/@azure/core-client/-/core-client-1.10.1.tgz", + "integrity": "sha512-Nh5PhEOeY6PrnxNPsEHRr9eimxLwgLlpmguQaHKBinFYA/RU9+kOYVOQqOrTsCL+KSxrLLl1gD8Dk5BFW/7l/w==", + "license": "MIT", "dependencies": { - "@azure/abort-controller": "^1.0.0", - "@azure/core-asynciterator-polyfill": "^1.0.0", - "@azure/core-auth": "^1.3.0", - "@azure/core-rest-pipeline": "^1.5.0", - "@azure/core-tracing": "1.0.0-preview.13", - "@azure/logger": "^1.0.0", - "tslib": "^2.2.0" + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.10.0", + "@azure/core-rest-pipeline": "^1.22.0", + "@azure/core-tracing": "^1.3.0", + "@azure/core-util": "^1.13.0", + "@azure/logger": "^1.3.0", + "tslib": "^2.6.2" }, "engines": { - "node": ">=12.0.0" + "node": ">=20.0.0" + } + }, + "node_modules/@azure/core-client/node_modules/@azure/abort-controller": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", + "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-client/node_modules/@azure/core-auth": { + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.10.1.tgz", + "integrity": "sha512-ykRMW8PjVAn+RS6ww5cmK9U2CyH9p4Q88YJwvUslfuMmN98w/2rdGRLPqJYObapBCdzBVeDgYWdJnFPFb7qzpg==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-util": "^1.13.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@azure/core-client/node_modules/@azure/core-rest-pipeline": { + "version": "1.23.0", + "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.23.0.tgz", + "integrity": "sha512-Evs1INHo+jUjwHi1T6SG6Ua/LHOQBCLuKEEE6efIpt4ZOoNonaT1kP32GoOcdNDbfqsD2445CPri3MubBy5DEQ==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.10.0", + "@azure/core-tracing": "^1.3.0", + "@azure/core-util": "^1.13.0", + "@azure/logger": "^1.3.0", + "@typespec/ts-http-runtime": "^0.3.4", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@azure/core-client/node_modules/@azure/core-tracing": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.3.1.tgz", + "integrity": "sha512-9MWKevR7Hz8kNzzPLfX4EAtGM2b8mr50HPDBvio96bURP/9C+HjdH3sBlLSNNrvRAr5/k/svoH457gB5IKpmwQ==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" } }, "node_modules/@azure/core-http": { @@ -245,11 +303,15 @@ } }, "node_modules/@azure/core-paging": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@azure/core-paging/-/core-paging-1.1.1.tgz", - "integrity": "sha512-hqEJBEGKan4YdOaL9ZG/GRG6PXaFd/Wb3SSjQW4LWotZzgl6xqG00h6wmkrpd2NNkbBkD1erLHBO3lPHApv+iQ==", + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/@azure/core-paging/-/core-paging-1.6.2.tgz", + "integrity": "sha512-YKWi9YuCU04B55h25cnOYZHxXYtEvQEbKST5vqRga7hWY9ydd3FZHdeQF8pyh+acWZvppw13M/LMGx0LABUVMA==", + "license": "MIT", "dependencies": { - "@azure/core-asynciterator-polyfill": "^1.0.0" + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" } }, "node_modules/@azure/core-rest-pipeline": { @@ -332,40 +394,43 @@ } }, "node_modules/@azure/core-util": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.11.0.tgz", - "integrity": "sha512-DxOSLua+NdpWoSqULhjDyAZTXFdP/LKkqtYuxxz1SCN289zk3OG8UOpnCQAz/tygyACBtWp/BoO72ptK7msY8g==", + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.13.1.tgz", + "integrity": "sha512-XPArKLzsvl0Hf0CaGyKHUyVgF7oDnhKoP85Xv6M4StF/1AhfORhZudHtOyf2s+FcbuQ9dPRAjB8J2KvRRMUK2A==", "license": "MIT", "dependencies": { - "@azure/abort-controller": "^2.0.0", + "@azure/abort-controller": "^2.1.2", + "@typespec/ts-http-runtime": "^0.3.0", "tslib": "^2.6.2" }, "engines": { - "node": ">=18.0.0" + "node": ">=20.0.0" } }, "node_modules/@azure/core-util/node_modules/@azure/abort-controller": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.0.0.tgz", - "integrity": "sha512-RP/mR/WJchR+g+nQFJGOec+nzeN/VvjlwbinccoqfhTsTHbb8X5+mLDp48kHT0ueyum0BNSwGm0kX0UZuIqTGg==", + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", + "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", + "license": "MIT", "dependencies": { - "tslib": "^2.2.0" + "tslib": "^2.6.2" }, "engines": { "node": ">=18.0.0" } }, "node_modules/@azure/core-xml": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@azure/core-xml/-/core-xml-1.2.0.tgz", - "integrity": "sha512-oWWQUWfllD3RO8Ixnsw5RjAUWPitjRI+LXSM0KFmgkSjl0R6RTQzXU2SEMsgAENkD5nzyI4yPpTRJcN2svM6ug==", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@azure/core-xml/-/core-xml-1.5.0.tgz", + "integrity": "sha512-D/sdlJBMJfx7gqoj66PKVmhDDaU6TKA49ptcolxdas29X7AfvLTmfAGLjAcIMBK7UZ2o4lygHIqVckOlQU3xWw==", "dev": true, + "license": "MIT", "dependencies": { - "fast-xml-parser": "^4.0.1", - "tslib": "^2.2.0" + "fast-xml-parser": "^5.0.7", + "tslib": "^2.8.1" }, "engines": { - "node": ">=12.0.0" + "node": ">=20.0.0" } }, "node_modules/@azure/data-tables": { @@ -481,18 +546,18 @@ } }, "node_modules/@azure/logger": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.0.0.tgz", - "integrity": "sha512-g2qLDgvmhyIxR3JVS8N67CyIOeFRKQlX/llxYJQr1OSGQqM3HTpVP8MjmjcEKbL/OIt2N9C9UFaNQuKOw1laOA==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.3.0.tgz", + "integrity": "sha512-fCqPIfOcLE+CGqGPd66c8bZpwAji98tZ4JI9i/mlTNTlsIWslCfpg48s/ypyLxZTump5sypjrKn2/kY7q8oAbA==", + "license": "MIT", "dependencies": { - "tslib": "^1.9.3" + "@typespec/ts-http-runtime": "^0.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" } }, - "node_modules/@azure/logger/node_modules/tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - }, "node_modules/@azure/ms-rest-js": { "version": "1.11.2", "resolved": "https://registry.npmjs.org/@azure/ms-rest-js/-/ms-rest-js-1.11.2.tgz", @@ -601,83 +666,307 @@ } }, "node_modules/@azure/storage-blob": { - "version": "12.16.0", - "resolved": "https://registry.npmjs.org/@azure/storage-blob/-/storage-blob-12.16.0.tgz", - "integrity": "sha512-jz33rUSUGUB65FgYrTRgRDjG6hdPHwfvHe+g/UrwVG8MsyLqSxg9TaW7Yuhjxu1v1OZ5xam2NU6+IpCN0xJO8Q==", + "version": "12.31.0", + "resolved": "https://registry.npmjs.org/@azure/storage-blob/-/storage-blob-12.31.0.tgz", + "integrity": "sha512-DBgNv10aCSxopt92DkTDD0o9xScXeBqPKGmR50FPZQaEcH4JLQ+GEOGEDv19V5BMkB7kxr+m4h6il/cCDPvmHg==", "dev": true, + "license": "MIT", "dependencies": { - "@azure/abort-controller": "^1.0.0", - "@azure/core-http": "^3.0.0", + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.9.0", + "@azure/core-client": "^1.9.3", + "@azure/core-http-compat": "^2.2.0", "@azure/core-lro": "^2.2.0", - "@azure/core-paging": "^1.1.1", - "@azure/core-tracing": "1.0.0-preview.13", - "@azure/logger": "^1.0.0", + "@azure/core-paging": "^1.6.2", + "@azure/core-rest-pipeline": "^1.19.1", + "@azure/core-tracing": "^1.2.0", + "@azure/core-util": "^1.11.0", + "@azure/core-xml": "^1.4.5", + "@azure/logger": "^1.1.4", + "@azure/storage-common": "^12.3.0", "events": "^3.0.0", - "tslib": "^2.2.0" + "tslib": "^2.8.1" }, "engines": { - "node": ">=14.0.0" + "node": ">=20.0.0" } }, - "node_modules/@azure/storage-blob/node_modules/@azure/core-http": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@azure/core-http/-/core-http-3.0.0.tgz", - "integrity": "sha512-BxI2SlGFPPz6J1XyZNIVUf0QZLBKFX+ViFjKOkzqD18J1zOINIQ8JSBKKr+i+v8+MB6LacL6Nn/sP/TE13+s2Q==", + "node_modules/@azure/storage-blob/node_modules/@azure/abort-controller": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", + "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", "dev": true, + "license": "MIT", "dependencies": { - "@azure/abort-controller": "^1.0.0", - "@azure/core-auth": "^1.3.0", - "@azure/core-tracing": "1.0.0-preview.13", - "@azure/core-util": "^1.1.1", - "@azure/logger": "^1.0.0", - "@types/node-fetch": "^2.5.0", - "@types/tunnel": "^0.0.3", - "form-data": "^4.0.0", - "node-fetch": "^2.6.7", - "process": "^0.11.10", - "tslib": "^2.2.0", - "tunnel": "^0.0.6", - "uuid": "^8.3.0", - "xml2js": "^0.4.19" + "tslib": "^2.6.2" }, "engines": { - "node": ">=14.0.0" + "node": ">=18.0.0" } }, - "node_modules/@azure/storage-blob/node_modules/form-data": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", - "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "node_modules/@azure/storage-blob/node_modules/@azure/core-auth": { + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.10.1.tgz", + "integrity": "sha512-ykRMW8PjVAn+RS6ww5cmK9U2CyH9p4Q88YJwvUslfuMmN98w/2rdGRLPqJYObapBCdzBVeDgYWdJnFPFb7qzpg==", "dev": true, + "license": "MIT", "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "mime-types": "^2.1.12" + "@azure/abort-controller": "^2.1.2", + "@azure/core-util": "^1.13.0", + "tslib": "^2.6.2" }, "engines": { - "node": ">= 6" + "node": ">=20.0.0" } }, - "node_modules/@azure/storage-blob/node_modules/uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "node_modules/@azure/storage-blob/node_modules/@azure/core-http-compat": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/@azure/core-http-compat/-/core-http-compat-2.3.2.tgz", + "integrity": "sha512-Tf6ltdKzOJEgxZeWLCjMxrxbodB/ZeCbzzA1A2qHbhzAjzjHoBVSUeSl/baT/oHAxhc4qdqVaDKnc2+iE932gw==", "dev": true, - "bin": { - "uuid": "dist/bin/uuid" + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "@azure/core-client": "^1.10.0", + "@azure/core-rest-pipeline": "^1.22.0" } }, - "node_modules/@azure/storage-blob/node_modules/xml2js": { - "version": "0.4.23", - "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz", - "integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==", + "node_modules/@azure/storage-blob/node_modules/@azure/core-rest-pipeline": { + "version": "1.23.0", + "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.23.0.tgz", + "integrity": "sha512-Evs1INHo+jUjwHi1T6SG6Ua/LHOQBCLuKEEE6efIpt4ZOoNonaT1kP32GoOcdNDbfqsD2445CPri3MubBy5DEQ==", "dev": true, + "license": "MIT", "dependencies": { - "sax": ">=0.6.0", - "xmlbuilder": "~11.0.0" + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.10.0", + "@azure/core-tracing": "^1.3.0", + "@azure/core-util": "^1.13.0", + "@azure/logger": "^1.3.0", + "@typespec/ts-http-runtime": "^0.3.4", + "tslib": "^2.6.2" }, "engines": { - "node": ">=4.0.0" + "node": ">=20.0.0" + } + }, + "node_modules/@azure/storage-blob/node_modules/@azure/core-tracing": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.3.1.tgz", + "integrity": "sha512-9MWKevR7Hz8kNzzPLfX4EAtGM2b8mr50HPDBvio96bURP/9C+HjdH3sBlLSNNrvRAr5/k/svoH457gB5IKpmwQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@azure/storage-common": { + "version": "12.3.0", + "resolved": "https://registry.npmjs.org/@azure/storage-common/-/storage-common-12.3.0.tgz", + "integrity": "sha512-/OFHhy86aG5Pe8dP5tsp+BuJ25JOAl9yaMU3WZbkeoiFMHFtJ7tu5ili7qEdBXNW9G5lDB19trwyI6V49F/8iQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.9.0", + "@azure/core-http-compat": "^2.2.0", + "@azure/core-rest-pipeline": "^1.19.1", + "@azure/core-tracing": "^1.2.0", + "@azure/core-util": "^1.11.0", + "@azure/logger": "^1.1.4", + "events": "^3.3.0", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@azure/storage-common/node_modules/@azure/abort-controller": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", + "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", + "dev": true, + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/storage-common/node_modules/@azure/core-auth": { + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.10.1.tgz", + "integrity": "sha512-ykRMW8PjVAn+RS6ww5cmK9U2CyH9p4Q88YJwvUslfuMmN98w/2rdGRLPqJYObapBCdzBVeDgYWdJnFPFb7qzpg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-util": "^1.13.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@azure/storage-common/node_modules/@azure/core-http-compat": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/@azure/core-http-compat/-/core-http-compat-2.3.2.tgz", + "integrity": "sha512-Tf6ltdKzOJEgxZeWLCjMxrxbodB/ZeCbzzA1A2qHbhzAjzjHoBVSUeSl/baT/oHAxhc4qdqVaDKnc2+iE932gw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "@azure/core-client": "^1.10.0", + "@azure/core-rest-pipeline": "^1.22.0" + } + }, + "node_modules/@azure/storage-common/node_modules/@azure/core-rest-pipeline": { + "version": "1.23.0", + "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.23.0.tgz", + "integrity": "sha512-Evs1INHo+jUjwHi1T6SG6Ua/LHOQBCLuKEEE6efIpt4ZOoNonaT1kP32GoOcdNDbfqsD2445CPri3MubBy5DEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.10.0", + "@azure/core-tracing": "^1.3.0", + "@azure/core-util": "^1.13.0", + "@azure/logger": "^1.3.0", + "@typespec/ts-http-runtime": "^0.3.4", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@azure/storage-common/node_modules/@azure/core-tracing": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.3.1.tgz", + "integrity": "sha512-9MWKevR7Hz8kNzzPLfX4EAtGM2b8mr50HPDBvio96bURP/9C+HjdH3sBlLSNNrvRAr5/k/svoH457gB5IKpmwQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@azure/storage-file-datalake": { + "version": "12.29.0", + "resolved": "https://registry.npmjs.org/@azure/storage-file-datalake/-/storage-file-datalake-12.29.0.tgz", + "integrity": "sha512-iNod3ugGFGvYJ2891UhSoICYu8iM8Q2jdub5nBzVWtMQGtr3mBRnzXK/cZeuMsF3i63yXZZmDQSvIzj7xWyObw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.9.0", + "@azure/core-client": "^1.9.3", + "@azure/core-http-compat": "^2.0.0", + "@azure/core-paging": "^1.6.2", + "@azure/core-rest-pipeline": "^1.19.1", + "@azure/core-tracing": "^1.2.0", + "@azure/core-util": "^1.11.0", + "@azure/core-xml": "^1.4.3", + "@azure/logger": "^1.1.4", + "@azure/storage-blob": "^12.30.0", + "@azure/storage-common": "^12.2.0", + "events": "^3.3.0", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@azure/storage-file-datalake/node_modules/@azure/abort-controller": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", + "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", + "dev": true, + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/storage-file-datalake/node_modules/@azure/core-auth": { + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.10.1.tgz", + "integrity": "sha512-ykRMW8PjVAn+RS6ww5cmK9U2CyH9p4Q88YJwvUslfuMmN98w/2rdGRLPqJYObapBCdzBVeDgYWdJnFPFb7qzpg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-util": "^1.13.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@azure/storage-file-datalake/node_modules/@azure/core-http-compat": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/@azure/core-http-compat/-/core-http-compat-2.3.2.tgz", + "integrity": "sha512-Tf6ltdKzOJEgxZeWLCjMxrxbodB/ZeCbzzA1A2qHbhzAjzjHoBVSUeSl/baT/oHAxhc4qdqVaDKnc2+iE932gw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "@azure/core-client": "^1.10.0", + "@azure/core-rest-pipeline": "^1.22.0" + } + }, + "node_modules/@azure/storage-file-datalake/node_modules/@azure/core-rest-pipeline": { + "version": "1.23.0", + "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.23.0.tgz", + "integrity": "sha512-Evs1INHo+jUjwHi1T6SG6Ua/LHOQBCLuKEEE6efIpt4ZOoNonaT1kP32GoOcdNDbfqsD2445CPri3MubBy5DEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.10.0", + "@azure/core-tracing": "^1.3.0", + "@azure/core-util": "^1.13.0", + "@azure/logger": "^1.3.0", + "@typespec/ts-http-runtime": "^0.3.4", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@azure/storage-file-datalake/node_modules/@azure/core-tracing": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.3.1.tgz", + "integrity": "sha512-9MWKevR7Hz8kNzzPLfX4EAtGM2b8mr50HPDBvio96bURP/9C+HjdH3sBlLSNNrvRAr5/k/svoH457gB5IKpmwQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" } }, "node_modules/@azure/storage-queue": { @@ -1808,6 +2097,42 @@ "url": "https://opencollective.com/typescript-eslint" } }, + "node_modules/@typespec/ts-http-runtime": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/@typespec/ts-http-runtime/-/ts-http-runtime-0.3.4.tgz", + "integrity": "sha512-CI0NhTrz4EBaa0U+HaaUZrJhPoso8sG7ZFya8uQoBA57fjzrjRSv87ekCjLZOFExN+gXE/z0xuN2QfH4H2HrLQ==", + "license": "MIT", + "dependencies": { + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@typespec/ts-http-runtime/node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/@typespec/ts-http-runtime/node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/@ungap/structured-clone": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz", @@ -5099,23 +5424,38 @@ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, - "node_modules/fast-xml-parser": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.5.0.tgz", - "integrity": "sha512-/PlTQCI96+fZMAOLMZK4CWG1ItCbfZ/0jx7UIJFChPNrx7tcEgerUgWbeieCM9MfHInUDyK8DWYZ+YrywDJuTg==", + "node_modules/fast-xml-builder": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.1.4.tgz", + "integrity": "sha512-f2jhpN4Eccy0/Uz9csxh3Nu6q4ErKxf0XIsasomfOihuSUa3/xw6w8dnOtCDgEItQFJG8KyXPzQXzcODDrrbOg==", "dev": true, "funding": [ { "type": "github", "url": "https://github.com/sponsors/NaturalIntelligence" - }, + } + ], + "license": "MIT", + "dependencies": { + "path-expression-matcher": "^1.1.3" + } + }, + "node_modules/fast-xml-parser": { + "version": "5.5.6", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.5.6.tgz", + "integrity": "sha512-3+fdZyBRVg29n4rXP0joHthhcHdPUHaIC16cuyyd1iLsuaO6Vea36MPrxgAzbZna8lhvZeRL8Bc9GP56/J9xEw==", + "dev": true, + "funding": [ { - "type": "paypal", - "url": "https://paypal.me/naturalintelligence" + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" } ], + "license": "MIT", "dependencies": { - "strnum": "^1.0.5" + "fast-xml-builder": "^1.1.4", + "path-expression-matcher": "^1.1.3", + "strnum": "^2.1.2" }, "bin": { "fxparser": "src/cli/cli.js" @@ -8069,6 +8409,22 @@ "node": ">= 0.8" } }, + "node_modules/path-expression-matcher": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.1.3.tgz", + "integrity": "sha512-qdVgY8KXmVdJZRSS1JdEPOKPdTiEK/pi0RkcT2sw1RhXxohdujUlJFPuS1TSkevZ9vzd3ZlL7ULl1MHGTApKzQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", @@ -9624,10 +9980,17 @@ } }, "node_modules/strnum": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", - "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==", - "dev": true + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.2.0.tgz", + "integrity": "sha512-Y7Bj8XyJxnPAORMZj/xltsfo55uOiyHcU2tnAVzHUnSJR/KsEX+9RoDeXEnsXtl/CX4fAcrt64gZ13aGaWPeBg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" }, "node_modules/supports-color": { "version": "5.5.0", @@ -9901,9 +10264,10 @@ } }, "node_modules/tslib": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", - "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" }, "node_modules/tsutils": { "version": "3.21.0", @@ -10684,17 +11048,59 @@ } }, "@azure/core-client": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/@azure/core-client/-/core-client-1.5.0.tgz", - "integrity": "sha512-YNk8i9LT6YcFdFO+RRU0E4Ef+A8Y5lhXo6lz61rwbG8Uo7kSqh0YqK04OexiilM43xd6n3Y9yBhLnb1NFNI9dA==", - "requires": { - "@azure/abort-controller": "^1.0.0", - "@azure/core-asynciterator-polyfill": "^1.0.0", - "@azure/core-auth": "^1.3.0", - "@azure/core-rest-pipeline": "^1.5.0", - "@azure/core-tracing": "1.0.0-preview.13", - "@azure/logger": "^1.0.0", - "tslib": "^2.2.0" + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/@azure/core-client/-/core-client-1.10.1.tgz", + "integrity": "sha512-Nh5PhEOeY6PrnxNPsEHRr9eimxLwgLlpmguQaHKBinFYA/RU9+kOYVOQqOrTsCL+KSxrLLl1gD8Dk5BFW/7l/w==", + "requires": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.10.0", + "@azure/core-rest-pipeline": "^1.22.0", + "@azure/core-tracing": "^1.3.0", + "@azure/core-util": "^1.13.0", + "@azure/logger": "^1.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "@azure/abort-controller": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", + "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@azure/core-auth": { + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.10.1.tgz", + "integrity": "sha512-ykRMW8PjVAn+RS6ww5cmK9U2CyH9p4Q88YJwvUslfuMmN98w/2rdGRLPqJYObapBCdzBVeDgYWdJnFPFb7qzpg==", + "requires": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-util": "^1.13.0", + "tslib": "^2.6.2" + } + }, + "@azure/core-rest-pipeline": { + "version": "1.23.0", + "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.23.0.tgz", + "integrity": "sha512-Evs1INHo+jUjwHi1T6SG6Ua/LHOQBCLuKEEE6efIpt4ZOoNonaT1kP32GoOcdNDbfqsD2445CPri3MubBy5DEQ==", + "requires": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.10.0", + "@azure/core-tracing": "^1.3.0", + "@azure/core-util": "^1.13.0", + "@azure/logger": "^1.3.0", + "@typespec/ts-http-runtime": "^0.3.4", + "tslib": "^2.6.2" + } + }, + "@azure/core-tracing": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.3.1.tgz", + "integrity": "sha512-9MWKevR7Hz8kNzzPLfX4EAtGM2b8mr50HPDBvio96bURP/9C+HjdH3sBlLSNNrvRAr5/k/svoH457gB5IKpmwQ==", + "requires": { + "tslib": "^2.6.2" + } + } } }, "@azure/core-http": { @@ -10767,11 +11173,11 @@ } }, "@azure/core-paging": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@azure/core-paging/-/core-paging-1.1.1.tgz", - "integrity": "sha512-hqEJBEGKan4YdOaL9ZG/GRG6PXaFd/Wb3SSjQW4LWotZzgl6xqG00h6wmkrpd2NNkbBkD1erLHBO3lPHApv+iQ==", + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/@azure/core-paging/-/core-paging-1.6.2.tgz", + "integrity": "sha512-YKWi9YuCU04B55h25cnOYZHxXYtEvQEbKST5vqRga7hWY9ydd3FZHdeQF8pyh+acWZvppw13M/LMGx0LABUVMA==", "requires": { - "@azure/core-asynciterator-polyfill": "^1.0.0" + "tslib": "^2.6.2" } }, "@azure/core-rest-pipeline": { @@ -10834,32 +11240,33 @@ } }, "@azure/core-util": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.11.0.tgz", - "integrity": "sha512-DxOSLua+NdpWoSqULhjDyAZTXFdP/LKkqtYuxxz1SCN289zk3OG8UOpnCQAz/tygyACBtWp/BoO72ptK7msY8g==", + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.13.1.tgz", + "integrity": "sha512-XPArKLzsvl0Hf0CaGyKHUyVgF7oDnhKoP85Xv6M4StF/1AhfORhZudHtOyf2s+FcbuQ9dPRAjB8J2KvRRMUK2A==", "requires": { - "@azure/abort-controller": "^2.0.0", + "@azure/abort-controller": "^2.1.2", + "@typespec/ts-http-runtime": "^0.3.0", "tslib": "^2.6.2" }, "dependencies": { "@azure/abort-controller": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.0.0.tgz", - "integrity": "sha512-RP/mR/WJchR+g+nQFJGOec+nzeN/VvjlwbinccoqfhTsTHbb8X5+mLDp48kHT0ueyum0BNSwGm0kX0UZuIqTGg==", + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", + "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", "requires": { - "tslib": "^2.2.0" + "tslib": "^2.6.2" } } } }, "@azure/core-xml": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@azure/core-xml/-/core-xml-1.2.0.tgz", - "integrity": "sha512-oWWQUWfllD3RO8Ixnsw5RjAUWPitjRI+LXSM0KFmgkSjl0R6RTQzXU2SEMsgAENkD5nzyI4yPpTRJcN2svM6ug==", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@azure/core-xml/-/core-xml-1.5.0.tgz", + "integrity": "sha512-D/sdlJBMJfx7gqoj66PKVmhDDaU6TKA49ptcolxdas29X7AfvLTmfAGLjAcIMBK7UZ2o4lygHIqVckOlQU3xWw==", "dev": true, "requires": { - "fast-xml-parser": "^4.0.1", - "tslib": "^2.2.0" + "fast-xml-parser": "^5.0.7", + "tslib": "^2.8.1" } }, "@azure/data-tables": { @@ -10961,18 +11368,12 @@ } }, "@azure/logger": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.0.0.tgz", - "integrity": "sha512-g2qLDgvmhyIxR3JVS8N67CyIOeFRKQlX/llxYJQr1OSGQqM3HTpVP8MjmjcEKbL/OIt2N9C9UFaNQuKOw1laOA==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.3.0.tgz", + "integrity": "sha512-fCqPIfOcLE+CGqGPd66c8bZpwAji98tZ4JI9i/mlTNTlsIWslCfpg48s/ypyLxZTump5sypjrKn2/kY7q8oAbA==", "requires": { - "tslib": "^1.9.3" - }, - "dependencies": { - "tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - } + "@typespec/ts-http-runtime": "^0.3.0", + "tslib": "^2.6.2" } }, "@azure/ms-rest-js": { @@ -11068,68 +11469,227 @@ } }, "@azure/storage-blob": { - "version": "12.16.0", - "resolved": "https://registry.npmjs.org/@azure/storage-blob/-/storage-blob-12.16.0.tgz", - "integrity": "sha512-jz33rUSUGUB65FgYrTRgRDjG6hdPHwfvHe+g/UrwVG8MsyLqSxg9TaW7Yuhjxu1v1OZ5xam2NU6+IpCN0xJO8Q==", + "version": "12.31.0", + "resolved": "https://registry.npmjs.org/@azure/storage-blob/-/storage-blob-12.31.0.tgz", + "integrity": "sha512-DBgNv10aCSxopt92DkTDD0o9xScXeBqPKGmR50FPZQaEcH4JLQ+GEOGEDv19V5BMkB7kxr+m4h6il/cCDPvmHg==", "dev": true, "requires": { - "@azure/abort-controller": "^1.0.0", - "@azure/core-http": "^3.0.0", + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.9.0", + "@azure/core-client": "^1.9.3", + "@azure/core-http-compat": "^2.2.0", "@azure/core-lro": "^2.2.0", - "@azure/core-paging": "^1.1.1", - "@azure/core-tracing": "1.0.0-preview.13", - "@azure/logger": "^1.0.0", + "@azure/core-paging": "^1.6.2", + "@azure/core-rest-pipeline": "^1.19.1", + "@azure/core-tracing": "^1.2.0", + "@azure/core-util": "^1.11.0", + "@azure/core-xml": "^1.4.5", + "@azure/logger": "^1.1.4", + "@azure/storage-common": "^12.3.0", "events": "^3.0.0", - "tslib": "^2.2.0" + "tslib": "^2.8.1" }, "dependencies": { - "@azure/core-http": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@azure/core-http/-/core-http-3.0.0.tgz", - "integrity": "sha512-BxI2SlGFPPz6J1XyZNIVUf0QZLBKFX+ViFjKOkzqD18J1zOINIQ8JSBKKr+i+v8+MB6LacL6Nn/sP/TE13+s2Q==", + "@azure/abort-controller": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", + "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", "dev": true, "requires": { - "@azure/abort-controller": "^1.0.0", - "@azure/core-auth": "^1.3.0", - "@azure/core-tracing": "1.0.0-preview.13", - "@azure/core-util": "^1.1.1", - "@azure/logger": "^1.0.0", - "@types/node-fetch": "^2.5.0", - "@types/tunnel": "^0.0.3", - "form-data": "^4.0.0", - "node-fetch": "^2.6.7", - "process": "^0.11.10", - "tslib": "^2.2.0", - "tunnel": "^0.0.6", - "uuid": "^8.3.0", - "xml2js": "^0.4.19" + "tslib": "^2.6.2" } }, - "form-data": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", - "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "@azure/core-auth": { + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.10.1.tgz", + "integrity": "sha512-ykRMW8PjVAn+RS6ww5cmK9U2CyH9p4Q88YJwvUslfuMmN98w/2rdGRLPqJYObapBCdzBVeDgYWdJnFPFb7qzpg==", "dev": true, "requires": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "mime-types": "^2.1.12" + "@azure/abort-controller": "^2.1.2", + "@azure/core-util": "^1.13.0", + "tslib": "^2.6.2" } }, - "uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "dev": true + "@azure/core-http-compat": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/@azure/core-http-compat/-/core-http-compat-2.3.2.tgz", + "integrity": "sha512-Tf6ltdKzOJEgxZeWLCjMxrxbodB/ZeCbzzA1A2qHbhzAjzjHoBVSUeSl/baT/oHAxhc4qdqVaDKnc2+iE932gw==", + "dev": true, + "requires": { + "@azure/abort-controller": "^2.1.2" + } }, - "xml2js": { - "version": "0.4.23", - "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz", - "integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==", + "@azure/core-rest-pipeline": { + "version": "1.23.0", + "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.23.0.tgz", + "integrity": "sha512-Evs1INHo+jUjwHi1T6SG6Ua/LHOQBCLuKEEE6efIpt4ZOoNonaT1kP32GoOcdNDbfqsD2445CPri3MubBy5DEQ==", "dev": true, "requires": { - "sax": ">=0.6.0", - "xmlbuilder": "~11.0.0" + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.10.0", + "@azure/core-tracing": "^1.3.0", + "@azure/core-util": "^1.13.0", + "@azure/logger": "^1.3.0", + "@typespec/ts-http-runtime": "^0.3.4", + "tslib": "^2.6.2" + } + }, + "@azure/core-tracing": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.3.1.tgz", + "integrity": "sha512-9MWKevR7Hz8kNzzPLfX4EAtGM2b8mr50HPDBvio96bURP/9C+HjdH3sBlLSNNrvRAr5/k/svoH457gB5IKpmwQ==", + "dev": true, + "requires": { + "tslib": "^2.6.2" + } + } + } + }, + "@azure/storage-common": { + "version": "12.3.0", + "resolved": "https://registry.npmjs.org/@azure/storage-common/-/storage-common-12.3.0.tgz", + "integrity": "sha512-/OFHhy86aG5Pe8dP5tsp+BuJ25JOAl9yaMU3WZbkeoiFMHFtJ7tu5ili7qEdBXNW9G5lDB19trwyI6V49F/8iQ==", + "dev": true, + "requires": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.9.0", + "@azure/core-http-compat": "^2.2.0", + "@azure/core-rest-pipeline": "^1.19.1", + "@azure/core-tracing": "^1.2.0", + "@azure/core-util": "^1.11.0", + "@azure/logger": "^1.1.4", + "events": "^3.3.0", + "tslib": "^2.8.1" + }, + "dependencies": { + "@azure/abort-controller": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", + "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", + "dev": true, + "requires": { + "tslib": "^2.6.2" + } + }, + "@azure/core-auth": { + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.10.1.tgz", + "integrity": "sha512-ykRMW8PjVAn+RS6ww5cmK9U2CyH9p4Q88YJwvUslfuMmN98w/2rdGRLPqJYObapBCdzBVeDgYWdJnFPFb7qzpg==", + "dev": true, + "requires": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-util": "^1.13.0", + "tslib": "^2.6.2" + } + }, + "@azure/core-http-compat": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/@azure/core-http-compat/-/core-http-compat-2.3.2.tgz", + "integrity": "sha512-Tf6ltdKzOJEgxZeWLCjMxrxbodB/ZeCbzzA1A2qHbhzAjzjHoBVSUeSl/baT/oHAxhc4qdqVaDKnc2+iE932gw==", + "dev": true, + "requires": { + "@azure/abort-controller": "^2.1.2" + } + }, + "@azure/core-rest-pipeline": { + "version": "1.23.0", + "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.23.0.tgz", + "integrity": "sha512-Evs1INHo+jUjwHi1T6SG6Ua/LHOQBCLuKEEE6efIpt4ZOoNonaT1kP32GoOcdNDbfqsD2445CPri3MubBy5DEQ==", + "dev": true, + "requires": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.10.0", + "@azure/core-tracing": "^1.3.0", + "@azure/core-util": "^1.13.0", + "@azure/logger": "^1.3.0", + "@typespec/ts-http-runtime": "^0.3.4", + "tslib": "^2.6.2" + } + }, + "@azure/core-tracing": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.3.1.tgz", + "integrity": "sha512-9MWKevR7Hz8kNzzPLfX4EAtGM2b8mr50HPDBvio96bURP/9C+HjdH3sBlLSNNrvRAr5/k/svoH457gB5IKpmwQ==", + "dev": true, + "requires": { + "tslib": "^2.6.2" + } + } + } + }, + "@azure/storage-file-datalake": { + "version": "12.29.0", + "resolved": "https://registry.npmjs.org/@azure/storage-file-datalake/-/storage-file-datalake-12.29.0.tgz", + "integrity": "sha512-iNod3ugGFGvYJ2891UhSoICYu8iM8Q2jdub5nBzVWtMQGtr3mBRnzXK/cZeuMsF3i63yXZZmDQSvIzj7xWyObw==", + "dev": true, + "requires": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.9.0", + "@azure/core-client": "^1.9.3", + "@azure/core-http-compat": "^2.0.0", + "@azure/core-paging": "^1.6.2", + "@azure/core-rest-pipeline": "^1.19.1", + "@azure/core-tracing": "^1.2.0", + "@azure/core-util": "^1.11.0", + "@azure/core-xml": "^1.4.3", + "@azure/logger": "^1.1.4", + "@azure/storage-blob": "^12.30.0", + "@azure/storage-common": "^12.2.0", + "events": "^3.3.0", + "tslib": "^2.8.1" + }, + "dependencies": { + "@azure/abort-controller": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", + "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", + "dev": true, + "requires": { + "tslib": "^2.6.2" + } + }, + "@azure/core-auth": { + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.10.1.tgz", + "integrity": "sha512-ykRMW8PjVAn+RS6ww5cmK9U2CyH9p4Q88YJwvUslfuMmN98w/2rdGRLPqJYObapBCdzBVeDgYWdJnFPFb7qzpg==", + "dev": true, + "requires": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-util": "^1.13.0", + "tslib": "^2.6.2" + } + }, + "@azure/core-http-compat": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/@azure/core-http-compat/-/core-http-compat-2.3.2.tgz", + "integrity": "sha512-Tf6ltdKzOJEgxZeWLCjMxrxbodB/ZeCbzzA1A2qHbhzAjzjHoBVSUeSl/baT/oHAxhc4qdqVaDKnc2+iE932gw==", + "dev": true, + "requires": { + "@azure/abort-controller": "^2.1.2" + } + }, + "@azure/core-rest-pipeline": { + "version": "1.23.0", + "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.23.0.tgz", + "integrity": "sha512-Evs1INHo+jUjwHi1T6SG6Ua/LHOQBCLuKEEE6efIpt4ZOoNonaT1kP32GoOcdNDbfqsD2445CPri3MubBy5DEQ==", + "dev": true, + "requires": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.10.0", + "@azure/core-tracing": "^1.3.0", + "@azure/core-util": "^1.13.0", + "@azure/logger": "^1.3.0", + "@typespec/ts-http-runtime": "^0.3.4", + "tslib": "^2.6.2" + } + }, + "@azure/core-tracing": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.3.1.tgz", + "integrity": "sha512-9MWKevR7Hz8kNzzPLfX4EAtGM2b8mr50HPDBvio96bURP/9C+HjdH3sBlLSNNrvRAr5/k/svoH457gB5IKpmwQ==", + "dev": true, + "requires": { + "tslib": "^2.6.2" } } } @@ -11981,6 +12541,32 @@ "eslint-visitor-keys": "^3.3.0" } }, + "@typespec/ts-http-runtime": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/@typespec/ts-http-runtime/-/ts-http-runtime-0.3.4.tgz", + "integrity": "sha512-CI0NhTrz4EBaa0U+HaaUZrJhPoso8sG7ZFya8uQoBA57fjzrjRSv87ekCjLZOFExN+gXE/z0xuN2QfH4H2HrLQ==", + "requires": { + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==" + }, + "https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "requires": { + "agent-base": "^7.1.2", + "debug": "4" + } + } + } + }, "@ungap/structured-clone": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz", @@ -14630,13 +15216,24 @@ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, + "fast-xml-builder": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.1.4.tgz", + "integrity": "sha512-f2jhpN4Eccy0/Uz9csxh3Nu6q4ErKxf0XIsasomfOihuSUa3/xw6w8dnOtCDgEItQFJG8KyXPzQXzcODDrrbOg==", + "dev": true, + "requires": { + "path-expression-matcher": "^1.1.3" + } + }, "fast-xml-parser": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.5.0.tgz", - "integrity": "sha512-/PlTQCI96+fZMAOLMZK4CWG1ItCbfZ/0jx7UIJFChPNrx7tcEgerUgWbeieCM9MfHInUDyK8DWYZ+YrywDJuTg==", + "version": "5.5.6", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.5.6.tgz", + "integrity": "sha512-3+fdZyBRVg29n4rXP0joHthhcHdPUHaIC16cuyyd1iLsuaO6Vea36MPrxgAzbZna8lhvZeRL8Bc9GP56/J9xEw==", "dev": true, "requires": { - "strnum": "^1.0.5" + "fast-xml-builder": "^1.1.4", + "path-expression-matcher": "^1.1.3", + "strnum": "^2.1.2" } }, "fastq": { @@ -16808,6 +17405,12 @@ "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==" }, + "path-expression-matcher": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.1.3.tgz", + "integrity": "sha512-qdVgY8KXmVdJZRSS1JdEPOKPdTiEK/pi0RkcT2sw1RhXxohdujUlJFPuS1TSkevZ9vzd3ZlL7ULl1MHGTApKzQ==", + "dev": true + }, "path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", @@ -17921,9 +18524,9 @@ "dev": true }, "strnum": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", - "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.2.0.tgz", + "integrity": "sha512-Y7Bj8XyJxnPAORMZj/xltsfo55uOiyHcU2tnAVzHUnSJR/KsEX+9RoDeXEnsXtl/CX4fAcrt64gZ13aGaWPeBg==", "dev": true }, "supports-color": { @@ -18136,9 +18739,9 @@ } }, "tslib": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", - "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==" }, "tsutils": { "version": "3.21.0", diff --git a/package.json b/package.json index 6df18cfcb..7afe7312e 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ "@azure/core-rest-pipeline": "^1.2.0", "@azure/data-tables": "^13.0.1", "@azure/storage-blob": "^12.9.0", + "@azure/storage-file-datalake": "^12.29.0", "@azure/storage-queue": "^12.8.0", "@types/args": "^5.0.0", "@types/async": "^3.0.1", @@ -208,6 +209,16 @@ "default": 10000, "description": "Blob service listening port, by default 10000" }, + "azurite.dfsHost": { + "type": "string", + "default": "127.0.0.1", + "description": "DFS service listening endpoint, by default 127.0.0.1" + }, + "azurite.dfsPort": { + "type": "number", + "default": 10004, + "description": "DFS service listening port, by default 10004" + }, "azurite.blobKeepAliveTimeout": { "type": "number", "default": 5, @@ -303,6 +314,7 @@ "build:autorest:blob": "autorest ./swagger/blob.md --typescript --use=S:/GitHub/XiaoningLiu/autorest.typescript.server", "build:autorest:queue": "autorest ./swagger/queue.md --typescript --use=S:/GitHub/XiaoningLiu/autorest.typescript.server", "build:autorest:table": "autorest ./swagger/table.md --typescript --use=S:/GitHub/XiaoningLiu/autorest.typescript.server", + "build:autorest:dfs": "autorest ./swagger/dfs.md --typescript --use=S:/GitHub/XiaoningLiu/autorest.typescript.server", "build:exe": "node ./scripts/buildExe.js", "build:linux": "node ./scripts/buildLinux.js", "watch": "tsc -watch -p ./", @@ -350,4 +362,4 @@ "url": "https://github.com/azure/azurite/issues" }, "homepage": "https://github.com/azure/azurite#readme" -} \ No newline at end of file +} diff --git a/src/azurite.ts b/src/azurite.ts index e856c3e25..2905e0a37 100644 --- a/src/azurite.ts +++ b/src/azurite.ts @@ -18,6 +18,8 @@ import { } from "./queue/utils/constants"; import SqlBlobServer from "./blob/SqlBlobServer"; import BlobServer from "./blob/BlobServer"; +import DfsServer from "./blob/DfsServer"; +import DfsConfiguration from "./blob/DfsConfiguration"; import TableConfiguration from "./table/TableConfiguration"; import TableServer from "./table/TableServer"; @@ -30,11 +32,14 @@ import { AzuriteTelemetryClient } from "./common/Telemetry"; function shutdown( blobServer: BlobServer | SqlBlobServer, + dfsServer: DfsServer, queueServer: QueueServer, tableServer: TableServer ) { const blobBeforeCloseMessage = `Azurite Blob service is closing...`; const blobAfterCloseMessage = `Azurite Blob service successfully closed`; + const dfsBeforeCloseMessage = `Azurite DFS service is closing...`; + const dfsAfterCloseMessage = `Azurite DFS service successfully closed`; const queueBeforeCloseMessage = `Azurite Queue service is closing...`; const queueAfterCloseMessage = `Azurite Queue service successfully closed`; const tableBeforeCloseMessage = `Azurite Table service is closing...`; @@ -47,6 +52,11 @@ function shutdown( console.log(blobAfterCloseMessage); }); + console.log(dfsBeforeCloseMessage); + dfsServer.close().then(() => { + console.log(dfsAfterCloseMessage); + }); + console.log(queueBeforeCloseMessage); queueServer.close().then(() => { console.log(queueAfterCloseMessage); @@ -79,6 +89,24 @@ async function main() { const blobServerFactory = new BlobServerFactory(); const blobServer = await blobServerFactory.createServer(env); const blobConfig = blobServer.config; + const dfsConfig = new DfsConfiguration( + env.dfsHost(), + env.dfsPort(), + env.blobKeepAliveTimeout(), + env.cert(), + env.key(), + env.pwd() + ); + const blobServerAny = blobServer as any; + const enableHns = env.enableHierarchicalNamespace(); + const dfsServer = new DfsServer( + dfsConfig, + blobServerAny.metadataStore, + blobServerAny.extentStore, + blobServerAny.accountDataStore, + undefined, + enableHns + ); // TODO: Align with blob DEFAULT_BLOB_PERSISTENCE_ARRAY // TODO: Join for all paths in the array @@ -150,6 +178,14 @@ async function main() { `Azurite Blob service is successfully listening at ${blobServer.getHttpServerAddress()}` ); + console.log( + `Azurite DFS service is starting at ${dfsConfig.getHttpServerAddress()}` + ); + await dfsServer.start(); + console.log( + `Azurite DFS service is successfully listening at ${dfsServer.getHttpServerAddress()}` + ); + // Start server console.log( `Azurite Queue service is starting at ${queueConfig.getHttpServerAddress()}` @@ -175,11 +211,11 @@ async function main() { process .once("message", (msg) => { if (msg === "shutdown") { - shutdown(blobServer, queueServer, tableServer); + shutdown(blobServer, dfsServer, queueServer, tableServer); } }) - .once("SIGINT", () => shutdown(blobServer, queueServer, tableServer)) - .once("SIGTERM", () => shutdown(blobServer, queueServer, tableServer)); + .once("SIGINT", () => shutdown(blobServer, dfsServer, queueServer, tableServer)) + .once("SIGTERM", () => shutdown(blobServer, dfsServer, queueServer, tableServer)); } main().catch((err) => { diff --git a/src/blob/BlobEnvironment.ts b/src/blob/BlobEnvironment.ts index 19978e592..8b82c4115 100644 --- a/src/blob/BlobEnvironment.ts +++ b/src/blob/BlobEnvironment.ts @@ -5,6 +5,8 @@ import { dirname } from "path"; import IBlobEnvironment from "./IBlobEnvironment"; import { DEFAULT_BLOB_LISTENING_PORT, + DEFAULT_DFS_LISTENING_PORT, + DEFAULT_DFS_SERVER_HOST_NAME, DEFAULT_BLOB_SERVER_HOST_NAME, DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT } from "./utils/constants"; @@ -21,6 +23,16 @@ if (!(args as any).config.name) { "Optional. Customize listening port for blob", DEFAULT_BLOB_LISTENING_PORT ) + .option( + ["", "dfsHost"], + "Optional. Customize listening address for DFS", + DEFAULT_DFS_SERVER_HOST_NAME + ) + .option( + ["", "dfsPort"], + "Optional. Customize listening port for DFS", + DEFAULT_DFS_LISTENING_PORT + ) .option( ["", "blobKeepAliveTimeout"], "Optional. Customize http keep alive timeout for blob", @@ -69,6 +81,10 @@ if (!(args as any).config.name) { .option( ["", "disableTelemetry"], "Optional. Disable telemetry data collection of this Azurite execution. By default, Azurite will collect telemetry data to help improve the product." + ) + .option( + ["", "enableHierarchicalNamespace"], + "Optional. Enable hierarchical namespace (HNS) mode for ADLS Gen2. Default is true." ); (args as any).config.name = "azurite-blob"; @@ -85,6 +101,14 @@ export default class BlobEnvironment implements IBlobEnvironment { return this.flags.blobPort; } + public dfsHost(): string | undefined { + return this.flags.dfsHost; + } + + public dfsPort(): number | undefined { + return this.flags.dfsPort; + } + public blobKeepAliveTimeout(): number | undefined { return this.flags.keepAliveTimeout; } @@ -169,6 +193,13 @@ export default class BlobEnvironment implements IBlobEnvironment { return this.flags.extentMemoryLimit; } + public enableHierarchicalNamespace(): boolean { + if (this.flags.enableHierarchicalNamespace !== undefined) { + return this.flags.enableHierarchicalNamespace !== false; + } + return true; // default enabled + } + public async debug(): Promise { if (typeof this.flags.debug === "string") { // Enable debug log to file diff --git a/src/blob/BlobServer.ts b/src/blob/BlobServer.ts index 2f0e10dc3..fd828ba71 100644 --- a/src/blob/BlobServer.ts +++ b/src/blob/BlobServer.ts @@ -39,10 +39,10 @@ const AFTER_CLOSE_MESSAGE = `Azurite Blob service successfully closed`; * @class Server */ export default class BlobServer extends ServerBase implements ICleaner { - private readonly metadataStore: IBlobMetadataStore; + public readonly metadataStore: IBlobMetadataStore; private readonly extentMetadataStore: IExtentMetadataStore; - private readonly extentStore: IExtentStore; - private readonly accountDataStore: IAccountDataStore; + public readonly extentStore: IExtentStore; + public readonly accountDataStore: IAccountDataStore; private readonly gcManager: IGCManager; /** diff --git a/src/blob/DfsConfiguration.ts b/src/blob/DfsConfiguration.ts new file mode 100644 index 000000000..0e810ae00 --- /dev/null +++ b/src/blob/DfsConfiguration.ts @@ -0,0 +1,34 @@ +import ConfigurationBase from "../common/ConfigurationBase"; +import { + DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT, + DEFAULT_DFS_LISTENING_PORT, + DEFAULT_DFS_SERVER_HOST_NAME +} from "./utils/constants"; + +export default class DfsConfiguration extends ConfigurationBase { + public constructor( + host: string = DEFAULT_DFS_SERVER_HOST_NAME, + port: number = DEFAULT_DFS_LISTENING_PORT, + keepAliveTimeout: number = DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT, + cert: string = "", + key: string = "", + pwd: string = "" + ) { + super( + host, + port, + keepAliveTimeout, + false, + undefined, + false, + undefined, + false, + false, + cert, + key, + pwd, + undefined, + false + ); + } +} diff --git a/src/blob/DfsRequestListenerFactory.ts b/src/blob/DfsRequestListenerFactory.ts new file mode 100644 index 000000000..4b8d907ee --- /dev/null +++ b/src/blob/DfsRequestListenerFactory.ts @@ -0,0 +1,192 @@ +import express from "express"; + +import IAccountDataStore from "../common/IAccountDataStore"; +import IRequestListenerFactory from "../common/IRequestListenerFactory"; +import logger from "../common/Logger"; +import IExtentStore from "../common/persistence/IExtentStore"; +import { OAuthLevel } from "../common/models"; +import { RequestListener } from "../common/ServerBase"; +import IBlobMetadataStore from "./persistence/IBlobMetadataStore"; +import createDfsContextMiddleware, { getDfsContext } from "./dfs/DfsContext"; +import { DfsOperation } from "./dfs/DfsOperation"; +import createDfsAuthenticationMiddleware from "./dfs/DfsAuthenticationMiddleware"; +import FilesystemHandler from "./dfs/handlers/FilesystemHandler"; +import PathHandler from "./dfs/handlers/PathHandler"; +import { sendDfsError, internalError } from "./dfs/DfsErrorFactory"; + +/* + * Generated DFS layer at src/blob/generated-dfs/ provides: + * - OpenAPI/Swagger spec: swagger/dfs-storage-2023-11-03.json + * - AutoRest config: swagger/dfs.md + * - Typed interfaces: generated-dfs/handlers/IFilesystemHandler, IPathHandler + * - Operation enum: generated-dfs/artifacts/operation.ts + * - Models: generated-dfs/artifacts/models.ts + * - Dispatch specs: generated-dfs/artifacts/specifications.ts + * - Handler mappers: generated-dfs/handlers/handlerMappers.ts + * + * The concrete handlers (FilesystemHandler, PathHandler) currently use Express + * req/res directly. A future refactor can adapt them to the generated + * (options, context) → response pattern with a deserializer/serializer + * middleware layer, matching the blob endpoint architecture exactly. + */ + +/** + * DfsRequestListenerFactory creates the Express application for the DFS endpoint. + * + * Architecture follows the generated interface pattern from `src/blob/generated-dfs/`: + * + * 1. Context middleware — extracts account/filesystem/path from URL + * 2. Dispatch middleware — matches request to DfsOperation + * 3. Authentication — reuses blob SharedKey/SAS/OAuth authenticators + * 4. Handler middleware — routes to handler method + * 5. Error middleware — DFS JSON error responses + * + * Handler implementations (FilesystemHandler, PathHandler) fulfill the contracts + * defined by the generated IFilesystemHandler and IPathHandler interfaces. + */ +export default class DfsRequestListenerFactory implements IRequestListenerFactory { + public constructor( + private readonly metadataStore: IBlobMetadataStore, + private readonly extentStore: IExtentStore, + private readonly accountDataStore: IAccountDataStore, + private readonly oauth?: OAuthLevel, + private readonly enableHierarchicalNamespace: boolean = true + ) {} + + public createRequestListener(): RequestListener { + const app = express().disable("x-powered-by"); + + const filesystemHandler = new FilesystemHandler(this.metadataStore, this.enableHierarchicalNamespace); + const pathHandler = new PathHandler(this.metadataStore, this.extentStore, this.oauth); + + // Parse raw body for append operations + app.use(express.raw({ type: "*/*", limit: "256mb" })); + + // 1. Parse DFS context (account, filesystem, path) + app.use(createDfsContextMiddleware()); + + // 2. Dispatch: determine DFS operation from request + app.use((req: express.Request, res: express.Response, next: express.NextFunction) => { + const ctx = getDfsContext(res); + const resource = req.query.resource as string | undefined; + const action = req.query.action as string | undefined; + const method = req.method.toUpperCase(); + + let operation: DfsOperation | undefined; + + if (resource === "account" && method === "GET") { + operation = DfsOperation.Filesystem_List; + } else if (resource === "filesystem") { + if (ctx.path) { + operation = DfsOperation.Filesystem_ListPaths; + } else { + switch (method) { + case "PUT": operation = DfsOperation.Filesystem_Create; break; + case "DELETE": operation = DfsOperation.Filesystem_Delete; break; + case "HEAD": operation = DfsOperation.Filesystem_GetProperties; break; + case "PATCH": operation = DfsOperation.Filesystem_SetProperties; break; + case "GET": operation = DfsOperation.Filesystem_ListPaths; break; + } + } + } else if (ctx.filesystem && ctx.path) { + const leaseAction = req.headers["x-ms-lease-action"] as string | undefined; + if (leaseAction) { + operation = DfsOperation.Path_Lease; + } else if (req.headers["x-ms-rename-source"] && method === "PUT") { + operation = DfsOperation.Path_Rename; + } else if (resource === "file" || resource === "directory") { + operation = DfsOperation.Path_Create; + } else if (method === "HEAD") { + operation = action === "getAccessControl" + ? DfsOperation.Path_GetAccessControl + : DfsOperation.Path_GetProperties; + } else if (method === "GET") { + operation = DfsOperation.Path_Read; + } else if (method === "DELETE") { + operation = DfsOperation.Path_Delete; + } else if (action) { + // PATCH with action (append, flush, setAccessControl, etc.) + operation = DfsOperation.Path_Update; + } else if (method === "PUT") { + operation = DfsOperation.Path_Create; + } else if (method === "PATCH") { + operation = DfsOperation.Path_Update; + } + } else if (ctx.filesystem && !ctx.path) { + switch (method) { + case "GET": operation = DfsOperation.Filesystem_ListPaths; break; + case "PUT": operation = DfsOperation.Filesystem_Create; break; + case "DELETE": operation = DfsOperation.Filesystem_Delete; break; + case "HEAD": operation = DfsOperation.Filesystem_GetProperties; break; + } + } + + if (operation) { + ctx.operation = operation; + } + + next(); + }); + + // 3. Authentication middleware + app.use(createDfsAuthenticationMiddleware( + this.accountDataStore, + this.metadataStore, + logger, + this.oauth + )); + + // 4. Route to handler + app.use(async (req: express.Request, res: express.Response, next: express.NextFunction) => { + try { + const ctx = getDfsContext(res); + const operation = ctx.operation; + + switch (operation) { + case DfsOperation.Filesystem_Create: + return await filesystemHandler.create(req, res); + case DfsOperation.Filesystem_Delete: + return await filesystemHandler.delete(req, res); + case DfsOperation.Filesystem_GetProperties: + return await filesystemHandler.getProperties(req, res); + case DfsOperation.Filesystem_List: + return await filesystemHandler.list(req, res); + case DfsOperation.Filesystem_SetProperties: + return await filesystemHandler.setProperties(req, res); + case DfsOperation.Filesystem_ListPaths: + return await pathHandler.listPaths(req, res); + case DfsOperation.Path_Create: + case DfsOperation.Path_Rename: + return await pathHandler.create(req, res); + case DfsOperation.Path_Delete: + return await pathHandler.delete(req, res); + case DfsOperation.Path_GetProperties: + case DfsOperation.Path_GetAccessControl: + return await pathHandler.getProperties(req, res); + case DfsOperation.Path_Read: + return await pathHandler.read(req, res); + case DfsOperation.Path_Update: + return await pathHandler.update(req, res); + case DfsOperation.Path_Lease: + return await pathHandler.lease(req, res); + default: + res.status(400).json({ + error: { + code: "UnsupportedOperation", + message: `The requested operation is not supported.` + } + }); + } + } catch (error: any) { + next(error); + } + }); + + // 5. Error handler + app.use((error: Error, _req: express.Request, res: express.Response, _next: express.NextFunction) => { + sendDfsError(res, internalError(error.message)); + }); + + return app; + } +} diff --git a/src/blob/DfsServer.ts b/src/blob/DfsServer.ts new file mode 100644 index 000000000..7a62ca81b --- /dev/null +++ b/src/blob/DfsServer.ts @@ -0,0 +1,49 @@ +import * as http from "http"; +import * as https from "https"; + +import IAccountDataStore from "../common/IAccountDataStore"; +import { CertOptions } from "../common/ConfigurationBase"; +import { OAuthLevel } from "../common/models"; +import IExtentStore from "../common/persistence/IExtentStore"; +import ServerBase from "../common/ServerBase"; +import DfsConfiguration from "./DfsConfiguration"; +import DfsRequestListenerFactory from "./DfsRequestListenerFactory"; +import IBlobMetadataStore from "./persistence/IBlobMetadataStore"; + +export default class DfsServer extends ServerBase { + public constructor( + configuration: DfsConfiguration, + metadataStore: IBlobMetadataStore, + extentStore: IExtentStore, + accountDataStore: IAccountDataStore, + oauth?: OAuthLevel, + enableHierarchicalNamespace: boolean = true + ) { + let httpServer; + const certOption = configuration.hasCert(); + switch (certOption) { + case CertOptions.PEM: + case CertOptions.PFX: + httpServer = https.createServer(configuration.getCert(certOption)!); + break; + default: + httpServer = http.createServer(); + } + + const requestListenerFactory = new DfsRequestListenerFactory( + metadataStore, + extentStore, + accountDataStore, + oauth, + enableHierarchicalNamespace + ); + + super( + configuration.host, + configuration.port, + httpServer, + requestListenerFactory, + configuration + ); + } +} diff --git a/src/blob/IBlobEnvironment.ts b/src/blob/IBlobEnvironment.ts index a57700759..29cbfefcd 100644 --- a/src/blob/IBlobEnvironment.ts +++ b/src/blob/IBlobEnvironment.ts @@ -1,6 +1,8 @@ export default interface IBlobEnvironment { blobHost(): string | undefined; blobPort(): number | undefined; + dfsHost(): string | undefined; + dfsPort(): number | undefined; blobKeepAliveTimeout(): number | undefined; location(): Promise; silent(): boolean; @@ -15,4 +17,5 @@ export default interface IBlobEnvironment { inMemoryPersistence(): boolean; extentMemoryLimit(): number | undefined; disableTelemetry(): boolean; + enableHierarchicalNamespace(): boolean; } diff --git a/src/blob/SqlBlobServer.ts b/src/blob/SqlBlobServer.ts index c0e07e6d3..806d964c7 100644 --- a/src/blob/SqlBlobServer.ts +++ b/src/blob/SqlBlobServer.ts @@ -36,10 +36,10 @@ const AFTER_CLOSE_MESSAGE = `Azurite Blob service successfully closed`; * @class Server */ export default class SqlBlobServer extends ServerBase { - private readonly metadataStore: IBlobMetadataStore; + public readonly metadataStore: IBlobMetadataStore; private readonly extentMetadataStore: IExtentMetadataStore; - private readonly extentStore: IExtentStore; - private readonly accountDataStore: IAccountDataStore; + public readonly extentStore: IExtentStore; + public readonly accountDataStore: IAccountDataStore; private readonly gcManager: IGCManager; /** diff --git a/src/blob/dfs/DfsAclEnforcer.ts b/src/blob/dfs/DfsAclEnforcer.ts new file mode 100644 index 000000000..d3e74e631 --- /dev/null +++ b/src/blob/dfs/DfsAclEnforcer.ts @@ -0,0 +1,213 @@ +/** + * ACL Enforcement for DFS (ADLS Gen2) operations. + * + * Phase III: When --oauth acl is enabled, checks the caller's identity + * against POSIX ACL entries stored on each path before allowing operations. + * + * ACL format follows Azure ADLS Gen2: + * "user::rwx,user:oid:r-x,group::r-x,other::---" + * + * Limitations (per wiki guidance): + * - No AAD group membership resolution + * - $superuser identity bypasses all ACL checks + * - Emulator mode (no identity) bypasses all checks + */ + +import { IDfsAuthenticatedIdentity } from "./DfsContext"; + +/** Required permission for an operation */ +export type AclPermission = "r" | "w" | "x"; + +/** + * Maps DFS operations to the minimum required permission. + */ +export function getRequiredPermission( + operationDescription: string +): AclPermission { + switch (operationDescription) { + case "read": + case "getProperties": + case "getAccessControl": + case "listPaths": + return "r"; + case "create": + case "delete": + case "update": + case "setAccessControl": + case "setProperties": + case "rename": + case "lease": + return "w"; + case "listChildren": + return "x"; + default: + return "r"; + } +} + +/** + * Parsed ACL entry. + * Format: "type:entityId:permissions" + * Examples: "user::rwx", "user:abc-123:r-x", "group::r--", "other::---" + */ +export interface AclEntry { + type: "user" | "group" | "mask" | "other"; + entityId: string; // empty string for default user/group/other + read: boolean; + write: boolean; + execute: boolean; +} + +/** + * Parse an ACL string into structured entries. + * ACL format: "user::rwx,user:abc:r-x,group::r-x,mask::rwx,other::---" + */ +export function parseAcl(aclString: string | undefined): AclEntry[] { + if (!aclString) return []; + + return aclString.split(",").filter(Boolean).map(entry => { + const parts = entry.split(":"); + if (parts.length < 3) return null; + + const type = parts[0] as AclEntry["type"]; + const entityId = parts[1]; + const perms = parts[2]; + + return { + type, + entityId, + read: perms.charAt(0) === "r", + write: perms.charAt(1) === "w", + execute: perms.charAt(2) === "x" + }; + }).filter((e): e is AclEntry => e !== null); +} + +/** + * Check if an ACL entry grants the required permission. + */ +function entryHasPermission(entry: AclEntry, permission: AclPermission): boolean { + switch (permission) { + case "r": return entry.read; + case "w": return entry.write; + case "x": return entry.execute; + } +} + +/** + * Result of an ACL check. + */ +export interface AclCheckResult { + allowed: boolean; + reason: string; +} + +/** + * Check whether the given identity is authorized for the required permission + * based on the path's ACL metadata. + * + * Algorithm follows the POSIX ACL evaluation order: + * 1. If owner matches identity → use owner permissions + * 2. If a named user entry matches identity → use that entry (masked) + * 3. If group matches → use group permissions (masked) + * 4. Fall through to other permissions + * + * Special cases: + * - $superuser always passes (emulator admin) + * - No identity (unauthenticated) always passes (emulator dev mode) + * - No ACL metadata → use default permissions (rwxr-x---) + */ +export function checkAcl( + identity: IDfsAuthenticatedIdentity | undefined, + owner: string | undefined, + group: string | undefined, + permissionsStr: string | undefined, + aclStr: string | undefined, + requiredPermission: AclPermission +): AclCheckResult { + // No identity = emulator/dev mode → bypass + if (!identity || (!identity.oid && !identity.upn)) { + return { allowed: true, reason: "No authenticated identity — emulator mode bypass" }; + } + + // $superuser bypasses all ACL checks + const effectiveOwner = owner || "$superuser"; + if (effectiveOwner === "$superuser") { + return { allowed: true, reason: "$superuser bypasses ACL checks" }; + } + + const callerId = identity.oid || identity.upn || ""; + + // Check if caller is the owner + if (callerId === effectiveOwner) { + // Use owner permissions from the permissions string (chars 0-2) + const perms = permissionsStr || "rwxr-x---"; + const ownerPerms: AclEntry = { + type: "user", + entityId: "", + read: perms.charAt(0) === "r", + write: perms.charAt(1) === "w", + execute: perms.charAt(2) === "x" + }; + if (entryHasPermission(ownerPerms, requiredPermission)) { + return { allowed: true, reason: "Owner permission granted" }; + } + return { allowed: false, reason: "Owner does not have required permission" }; + } + + // Parse ACL entries for named user/group matching + const aclEntries = parseAcl(aclStr); + + // Find mask entry (used to limit named user and group permissions) + const maskEntry = aclEntries.find(e => e.type === "mask" && e.entityId === ""); + + // Check named user entries + const namedUser = aclEntries.find( + e => e.type === "user" && e.entityId !== "" && e.entityId === callerId + ); + if (namedUser) { + const effective = maskEntry + ? entryHasPermission(namedUser, requiredPermission) && entryHasPermission(maskEntry, requiredPermission) + : entryHasPermission(namedUser, requiredPermission); + if (effective) { + return { allowed: true, reason: `Named user ACL entry matched (${callerId})` }; + } + return { allowed: false, reason: `Named user ACL entry matched but lacks permission` }; + } + + // Check group (we can't resolve AD group membership per wiki constraints, + // so we only check the owning group if the caller matches it) + const effectiveGroup = group || "$superuser"; + if (callerId === effectiveGroup) { + const perms = permissionsStr || "rwxr-x---"; + const groupPerms: AclEntry = { + type: "group", + entityId: "", + read: perms.charAt(3) === "r", + write: perms.charAt(4) === "w", + execute: perms.charAt(5) === "x" + }; + const effective = maskEntry + ? entryHasPermission(groupPerms, requiredPermission) && entryHasPermission(maskEntry, requiredPermission) + : entryHasPermission(groupPerms, requiredPermission); + if (effective) { + return { allowed: true, reason: "Group permission granted" }; + } + return { allowed: false, reason: "Group does not have required permission" }; + } + + // Fall through to "other" permissions (chars 6-8) + const perms = permissionsStr || "rwxr-x---"; + const otherPerms: AclEntry = { + type: "other", + entityId: "", + read: perms.charAt(6) === "r", + write: perms.charAt(7) === "w", + execute: perms.charAt(8) === "x" + }; + if (entryHasPermission(otherPerms, requiredPermission)) { + return { allowed: true, reason: "Other permission granted" }; + } + + return { allowed: false, reason: "Insufficient ACL permissions" }; +} diff --git a/src/blob/dfs/DfsAuthenticationMiddleware.ts b/src/blob/dfs/DfsAuthenticationMiddleware.ts new file mode 100644 index 000000000..fe877e72b --- /dev/null +++ b/src/blob/dfs/DfsAuthenticationMiddleware.ts @@ -0,0 +1,168 @@ +import { decode } from "jsonwebtoken"; +import { NextFunction, Request, RequestHandler, Response } from "express"; + +import IAccountDataStore from "../../common/IAccountDataStore"; +import ILogger from "../../common/ILogger"; +import IAuthenticator from "../authentication/IAuthenticator"; +import AccountSASAuthenticator from "../authentication/AccountSASAuthenticator"; +import BlobSASAuthenticator from "../authentication/BlobSASAuthenticator"; +import BlobSharedKeyAuthenticator from "../authentication/BlobSharedKeyAuthenticator"; +import BlobTokenAuthenticator from "../authentication/BlobTokenAuthenticator"; +import BlobStorageContext from "../context/BlobStorageContext"; +import ExpressRequestAdapter from "../generated/ExpressRequestAdapter"; + +import Operation from "../generated/artifacts/operation"; +import IBlobMetadataStore from "../persistence/IBlobMetadataStore"; +import { getDfsContext, IDfsAuthenticatedIdentity } from "./DfsContext"; +import { DfsOperation } from "./DfsOperation"; +import { sendDfsError } from "./DfsErrorFactory"; +import { OAuthLevel } from "../../common/models"; +import { BEARER_TOKEN_PREFIX } from "../../common/utils/constants"; + +const DEFAULT_CONTEXT_PATH = "dfs_blob_context"; + +/** + * Maps DFS operations to blob operations for SAS permission checking. + */ +function mapDfsOperationToBlobOperation(op?: DfsOperation): Operation { + switch (op) { + case DfsOperation.Filesystem_Create: + return Operation.Container_Create; + case DfsOperation.Filesystem_Delete: + return Operation.Container_Delete; + case DfsOperation.Filesystem_GetProperties: + return Operation.Container_GetProperties; + case DfsOperation.Filesystem_SetProperties: + return Operation.Container_SetMetadata; + case DfsOperation.Filesystem_List: + return Operation.Service_ListContainersSegment; + case DfsOperation.Filesystem_ListPaths: + return Operation.Container_ListBlobHierarchySegment; + case DfsOperation.Path_Create: + case DfsOperation.Path_Rename: + return Operation.BlockBlob_Upload; + case DfsOperation.Path_Delete: + return Operation.Blob_Delete; + case DfsOperation.Path_GetProperties: + case DfsOperation.Path_GetAccessControl: + return Operation.Blob_GetProperties; + case DfsOperation.Path_Read: + return Operation.Blob_Download; + case DfsOperation.Path_Update: + return Operation.BlockBlob_StageBlock; + case DfsOperation.Path_Lease: + return Operation.Blob_AcquireLease; + default: + return Operation.Blob_GetProperties; + } +} + +/** + * Extracts identity claims from a Bearer JWT token. + * Returns undefined if the token is not a Bearer token or can't be decoded. + */ +function extractIdentityFromRequest(req: Request): IDfsAuthenticatedIdentity | undefined { + const authHeader = req.header("authorization"); + if (!authHeader || !authHeader.startsWith(BEARER_TOKEN_PREFIX)) { + return undefined; + } + + const token = authHeader.substring(BEARER_TOKEN_PREFIX.length + 1); + try { + const decoded = decode(token) as { [key: string]: any } | null; + if (!decoded) return undefined; + + return { + oid: decoded.oid as string | undefined, + upn: decoded.upn as string | undefined, + tid: decoded.tid as string | undefined, + appid: decoded.appid as string | undefined + }; + } catch { + return undefined; + } +} + +export default function createDfsAuthenticationMiddleware( + accountDataStore: IAccountDataStore, + metadataStore: IBlobMetadataStore, + logger: ILogger, + oauth?: OAuthLevel +): RequestHandler { + const authenticators: IAuthenticator[] = [ + new BlobSharedKeyAuthenticator(accountDataStore, logger), + new AccountSASAuthenticator(accountDataStore, metadataStore, logger), + new BlobSASAuthenticator(accountDataStore, metadataStore, logger) + ]; + if (oauth !== undefined) { + authenticators.push( + new BlobTokenAuthenticator(accountDataStore, oauth, logger) + ); + } + + return async (req: Request, res: Response, next: NextFunction) => { + const dfsCtx = getDfsContext(res); + + // Build a BlobStorageContext that the existing authenticators can use + const holder: any = {}; + const blobContext = new BlobStorageContext(holder, DEFAULT_CONTEXT_PATH); + blobContext.startTime = dfsCtx.startTime; + blobContext.contextId = dfsCtx.requestId; + blobContext.account = dfsCtx.account; + blobContext.container = dfsCtx.filesystem; + blobContext.blob = dfsCtx.path; + blobContext.authenticationPath = dfsCtx.authenticationPath; + blobContext.loose = true; // DFS operations use loose mode for SAS validation + + // Set the blob operation for SAS permission checking + blobContext.operation = mapDfsOperationToBlobOperation(dfsCtx.operation); + + const request = new ExpressRequestAdapter(req); + + try { + let pass = false; + for (const authenticator of authenticators) { + const result = await authenticator.validate(request, blobContext); + if (result === true) { + pass = true; + break; + } + } + + if (!pass) { + // Check if there's no auth header at all — allow for dev/test + const hasAuth = req.header("authorization") !== undefined; + const hasSas = req.query.sig !== undefined; + if (!hasAuth && !hasSas) { + // No authentication provided — pass through (emulator mode) + return next(); + } + + sendDfsError(res, { + statusCode: 403, + code: "AuthorizationFailure", + message: "Server failed to authenticate the request." + }); + return; + } + + // When ACL mode is enabled, extract identity from bearer token + // so ACL enforcement can check permissions downstream + if (oauth === OAuthLevel.ACL) { + dfsCtx.identity = extractIdentityFromRequest(req); + } + + next(); + } catch (error: any) { + if (error.statusCode) { + sendDfsError(res, { + statusCode: error.statusCode, + code: error.storageErrorCode || "AuthenticationFailed", + message: error.storageErrorMessage || error.message + }); + } else { + next(error); + } + } + }; +} diff --git a/src/blob/dfs/DfsContext.ts b/src/blob/dfs/DfsContext.ts new file mode 100644 index 000000000..a8ae60f0f --- /dev/null +++ b/src/blob/dfs/DfsContext.ts @@ -0,0 +1,150 @@ +import uuid from "uuid/v4"; +import { NextFunction, Request, RequestHandler, Response } from "express"; + +import logger from "../../common/Logger"; +import { IP_REGEX, NO_ACCOUNT_HOST_NAMES } from "../../common/utils/constants"; +import { SECONDARY_SUFFIX, HeaderConstants, ValidAPIVersions, VERSION } from "../utils/constants"; +import { checkApiVersion } from "../utils/utils"; +import { DfsOperation } from "./DfsOperation"; + +/** + * Identity extracted from an OAuth bearer token. + * Used for ACL enforcement in Phase III (--oauth acl). + */ +export interface IDfsAuthenticatedIdentity { + /** Azure AD object ID (oid claim) */ + oid?: string; + /** User principal name (upn claim) */ + upn?: string; + /** Tenant ID (tid claim) */ + tid?: string; + /** Application ID (appid claim) */ + appid?: string; +} + +export interface IDfsContext { + requestId: string; + startTime: Date; + account?: string; + filesystem?: string; + path?: string; + isSecondary?: boolean; + operation?: DfsOperation; + authenticationPath?: string; + /** Authenticated identity from OAuth token — populated when --oauth acl is enabled */ + identity?: IDfsAuthenticatedIdentity; +} + +const DFS_CONTEXT_KEY = "dfsContext"; + +export function getDfsContext(res: Response): IDfsContext { + return res.locals[DFS_CONTEXT_KEY]; +} + +export default function createDfsContextMiddleware( + skipApiVersionCheck?: boolean, + disableProductStyleUrl?: boolean +): RequestHandler { + return (req: Request, res: Response, next: NextFunction) => { + res.setHeader(HeaderConstants.SERVER, `Azurite-DFS/${VERSION}`); + const requestId = uuid(); + + if (!skipApiVersionCheck) { + const apiVersion = req.header(HeaderConstants.X_MS_VERSION); + if (apiVersion !== undefined) { + checkApiVersion(apiVersion, ValidAPIVersions, requestId); + } + } + + const context: IDfsContext = { + requestId, + startTime: new Date() + }; + + const [account, filesystem, path, isSecondary] = extractDfsPartsFromPath( + req.hostname, + req.path, + disableProductStyleUrl + ); + + context.account = account; + context.filesystem = filesystem; + context.path = path; + context.isSecondary = isSecondary; + context.authenticationPath = req.path; + + if (isSecondary && context.authenticationPath) { + const pos = context.authenticationPath.search(SECONDARY_SUFFIX); + if (pos !== -1) { + context.authenticationPath = + context.authenticationPath.substr(0, pos) + + context.authenticationPath.substr(pos + SECONDARY_SUFFIX.length); + } + } + + res.locals[DFS_CONTEXT_KEY] = context; + + logger.info( + `DfsContextMiddleware: RequestMethod=${req.method} RequestURL=${req.protocol}://${req.hostname}${req.url} ClientIP=${req.ip}`, + requestId + ); + logger.info( + `DfsContextMiddleware: Account=${account} Filesystem=${filesystem} Path=${path}`, + requestId + ); + + if (!account) { + return res.status(400).json({ + error: { code: "InvalidQueryParameterValue", message: "Account name is required." } + }); + } + + next(); + }; +} + +function extractDfsPartsFromPath( + hostname: string, + path: string, + disableProductStyleUrl?: boolean +): [string | undefined, string | undefined, string | undefined, boolean] { + let account: string | undefined; + let filesystem: string | undefined; + let blobPath: string | undefined; + let isSecondary = false; + + const decodedPath = decodeURIComponent(path); + const normalizedPath = decodedPath.startsWith("/") + ? decodedPath.substr(1) + : decodedPath; + + const parts = normalizedPath.split("/"); + let urlPartIndex = 0; + + const isIPAddress = IP_REGEX.test(hostname); + const isNoAccountHostName = NO_ACCOUNT_HOST_NAMES.has(hostname.toLowerCase()); + const firstDotIndex = hostname.indexOf("."); + + if (!disableProductStyleUrl && !isIPAddress && !isNoAccountHostName && firstDotIndex > 0) { + account = hostname.substring(0, firstDotIndex); + } else { + account = parts[urlPartIndex++]; + } + + filesystem = parts[urlPartIndex++]; + blobPath = parts + .slice(urlPartIndex) + .join("/") + .replace(/\\/g, "/"); + + if (account && account.endsWith(SECONDARY_SUFFIX)) { + account = account.substr(0, account.length - SECONDARY_SUFFIX.length); + isSecondary = true; + } + + // Empty strings become undefined + if (!filesystem) filesystem = undefined; + if (!blobPath) blobPath = undefined; + + return [account, filesystem, blobPath, isSecondary]; +} diff --git a/src/blob/dfs/DfsContextFactory.ts b/src/blob/dfs/DfsContextFactory.ts new file mode 100644 index 000000000..ac477e193 --- /dev/null +++ b/src/blob/dfs/DfsContextFactory.ts @@ -0,0 +1,14 @@ +import Context from "../generated/Context"; + +/** + * Creates a minimal Context object suitable for passing to IBlobMetadataStore methods. + * DFS handlers don't go through the generated middleware pipeline, so we create + * context objects manually with the required fields. + */ +export function createStorageContext(requestId?: string): Context { + const holder: any = {}; + const ctx = new Context(holder, "ctx"); + ctx.startTime = new Date(); + ctx.contextId = requestId; + return ctx; +} diff --git a/src/blob/dfs/DfsErrorFactory.ts b/src/blob/dfs/DfsErrorFactory.ts new file mode 100644 index 000000000..df6757bc7 --- /dev/null +++ b/src/blob/dfs/DfsErrorFactory.ts @@ -0,0 +1,120 @@ +import { Response } from "express"; + +export interface DfsError { + statusCode: number; + code: string; + message: string; +} + +export function sendDfsError(res: Response, error: DfsError): void { + res.status(error.statusCode); + res.setHeader("x-ms-error-code", error.code); + + // HEAD requests must not include a response body. Sending Content-Type: application/json + // with an empty body causes Azure SDKs to crash trying to parse JSON from nothing. + // Use res.req (set by express) to detect HEAD without requiring callers to pass req. + if (res.req && res.req.method === "HEAD") { + res.setHeader("Content-Length", "0"); + res.end(); + } else { + res.json({ + error: { code: error.code, message: error.message } + }); + } +} + +export function filesystemNotFound(filesystem: string): DfsError { + return { + statusCode: 404, + code: "FilesystemNotFound", + message: `The specified filesystem does not exist. Filesystem: ${filesystem}` + }; +} + +export function pathNotFound(path: string): DfsError { + return { + statusCode: 404, + code: "PathNotFound", + message: `The specified path does not exist. Path: ${path}` + }; +} + +export function pathAlreadyExists(path: string): DfsError { + return { + statusCode: 409, + code: "PathAlreadyExists", + message: `The specified path already exists. Path: ${path}` + }; +} + +export function directoryNotEmpty(path: string): DfsError { + return { + statusCode: 409, + code: "DirectoryNotEmpty", + message: `The recursive query parameter value must be true to delete a non-empty directory. Path: ${path}` + }; +} + +export function invalidSourceOrDestination(message: string): DfsError { + return { + statusCode: 400, + code: "InvalidSourceUri", + message + }; +} + +export function invalidFlushPosition(): DfsError { + return { + statusCode: 400, + code: "InvalidFlushPosition", + message: "The uploaded data is not contiguous or the position query parameter value is not equal to the length of the file after appending the uploaded data." + }; +} + +export function conditionNotMet(): DfsError { + return { + statusCode: 412, + code: "ConditionNotMet", + message: "The condition specified using HTTP conditional header(s) is not met." + }; +} + +export function leaseIdMissing(): DfsError { + return { + statusCode: 412, + code: "LeaseIdMissing", + message: "There is currently a lease on the resource and no lease ID was specified in the request." + }; +} + +export function leaseNotPresent(): DfsError { + return { + statusCode: 409, + code: "LeaseNotPresentWithLeaseOperation", + message: "There is currently no lease on the resource." + }; +} + +export function leaseAlreadyPresent(): DfsError { + return { + statusCode: 409, + code: "LeaseAlreadyPresent", + message: "There is already a lease present." + }; +} + +export function leaseIdMismatch(): DfsError { + return { + statusCode: 409, + code: "LeaseIdMismatchWithLeaseOperation", + message: "The lease ID specified did not match the lease ID for the resource." + }; +} + +export function internalError(message: string): DfsError { + return { + statusCode: 500, + code: "InternalError", + message + }; +} diff --git a/src/blob/dfs/DfsOperation.ts b/src/blob/dfs/DfsOperation.ts new file mode 100644 index 000000000..6cfa416b4 --- /dev/null +++ b/src/blob/dfs/DfsOperation.ts @@ -0,0 +1,16 @@ +export enum DfsOperation { + Filesystem_Create = "Filesystem_Create", + Filesystem_Delete = "Filesystem_Delete", + Filesystem_GetProperties = "Filesystem_GetProperties", + Filesystem_SetProperties = "Filesystem_SetProperties", + Filesystem_List = "Filesystem_List", + Filesystem_ListPaths = "Filesystem_ListPaths", + Path_Create = "Path_Create", + Path_Delete = "Path_Delete", + Path_GetProperties = "Path_GetProperties", + Path_GetAccessControl = "Path_GetAccessControl", + Path_Read = "Path_Read", + Path_Update = "Path_Update", + Path_Rename = "Path_Rename", + Path_Lease = "Path_Lease" +} diff --git a/src/blob/dfs/DfsPropertyEncoding.ts b/src/blob/dfs/DfsPropertyEncoding.ts new file mode 100644 index 000000000..5db359b2f --- /dev/null +++ b/src/blob/dfs/DfsPropertyEncoding.ts @@ -0,0 +1,26 @@ +/** + * DFS uses x-ms-properties header with base64-encoded key=value pairs. + * Format: "key1=base64(value1),key2=base64(value2)" + */ + +export function encodeProperties(metadata: { [key: string]: string }): string { + return Object.entries(metadata) + .map(([key, value]) => `${key}=${Buffer.from(value).toString("base64")}`) + .join(","); +} + +export function decodeProperties(encoded: string): { [key: string]: string } { + const result: { [key: string]: string } = {}; + if (!encoded) return result; + + const pairs = encoded.split(","); + for (const pair of pairs) { + const eqIdx = pair.indexOf("="); + if (eqIdx >= 0) { + const key = pair.substring(0, eqIdx); + const value = Buffer.from(pair.substring(eqIdx + 1), "base64").toString("utf8"); + result[key] = value; + } + } + return result; +} diff --git a/src/blob/dfs/handlers/FilesystemHandler.ts b/src/blob/dfs/handlers/FilesystemHandler.ts new file mode 100644 index 000000000..65a1fa36f --- /dev/null +++ b/src/blob/dfs/handlers/FilesystemHandler.ts @@ -0,0 +1,219 @@ +import { Request, Response } from "express"; + +import logger from "../../../common/Logger"; +import IBlobMetadataStore from "../../persistence/IBlobMetadataStore"; +import { getDfsContext } from "../DfsContext"; +import { createStorageContext } from "../DfsContextFactory"; +import { sendDfsError, filesystemNotFound, internalError } from "../DfsErrorFactory"; +import { EMULATOR_ACCOUNT_NAME, BLOB_API_VERSION } from "../../utils/constants"; +import * as Models from "../../generated/artifacts/models"; + +export default class FilesystemHandler { + public constructor( + private readonly metadataStore: IBlobMetadataStore, + private readonly enableHierarchicalNamespace: boolean = true + ) {} + + public async create(req: Request, res: Response): Promise { + const ctx = getDfsContext(res); + const account = ctx.account || EMULATOR_ACCOUNT_NAME; + const filesystem = ctx.filesystem!; + const now = new Date(); + const etag = `"${now.getTime().toString(16)}"`; + + try { + const result = await this.metadataStore.createContainer(createStorageContext(ctx.requestId), { + accountName: account, + name: filesystem, + metadata: this.extractMetadata(req), + properties: { + lastModified: now, + etag, + leaseStatus: Models.LeaseStatusType.Unlocked, + leaseState: Models.LeaseStateType.Available, + hasImmutabilityPolicy: false, + hasLegalHold: false + } + } as any); + + res.status(201); + res.setHeader("ETag", result.properties.etag); + res.setHeader("Last-Modified", result.properties.lastModified.toUTCString()); + res.setHeader("x-ms-request-id", ctx.requestId); + res.setHeader("x-ms-version", BLOB_API_VERSION); + res.setHeader("x-ms-namespace-enabled", String(this.enableHierarchicalNamespace)); + res.end(); + } catch (error: any) { + if (error.statusCode === 409) { + return sendDfsError(res, { + statusCode: 409, + code: "FilesystemAlreadyExists", + message: `The specified filesystem already exists.` + }); + } + logger.error(`FilesystemHandler.create error: ${error.message}`, ctx.requestId); + sendDfsError(res, internalError(error.message)); + } + } + + public async delete(req: Request, res: Response): Promise { + const ctx = getDfsContext(res); + const account = ctx.account || EMULATOR_ACCOUNT_NAME; + const filesystem = ctx.filesystem!; + + try { + await this.metadataStore.deleteContainer( + createStorageContext(ctx.requestId), + account, + filesystem + ); + + res.status(202); + res.setHeader("x-ms-request-id", ctx.requestId); + res.setHeader("x-ms-version", BLOB_API_VERSION); + res.end(); + } catch (error: any) { + if (error.statusCode === 404) { + return sendDfsError(res, filesystemNotFound(filesystem)); + } + logger.error(`FilesystemHandler.delete error: ${error.message}`, ctx.requestId); + sendDfsError(res, internalError(error.message)); + } + } + + public async getProperties(req: Request, res: Response): Promise { + const ctx = getDfsContext(res); + const account = ctx.account || EMULATOR_ACCOUNT_NAME; + const filesystem = ctx.filesystem!; + + try { + const result = await this.metadataStore.getContainerProperties( + createStorageContext(ctx.requestId), + account, + filesystem + ); + + res.status(200); + res.setHeader("ETag", result.properties.etag); + res.setHeader("Last-Modified", result.properties.lastModified.toUTCString()); + res.setHeader("x-ms-request-id", ctx.requestId); + res.setHeader("x-ms-version", BLOB_API_VERSION); + res.setHeader("x-ms-resource-type", "filesystem"); + res.setHeader("x-ms-namespace-enabled", String(this.enableHierarchicalNamespace)); + + if (result.metadata) { + for (const [key, value] of Object.entries(result.metadata)) { + res.setHeader(`x-ms-properties-${key}`, Buffer.from(value).toString("base64")); + } + } + + res.end(); + } catch (error: any) { + if (error.statusCode === 404) { + return sendDfsError(res, filesystemNotFound(filesystem)); + } + logger.error(`FilesystemHandler.getProperties error: ${error.message}`, ctx.requestId); + sendDfsError(res, internalError(error.message)); + } + } + + public async list(req: Request, res: Response): Promise { + const ctx = getDfsContext(res); + const account = ctx.account || EMULATOR_ACCOUNT_NAME; + + const prefix = req.query.prefix as string | undefined; + const continuation = req.query.continuation as string | undefined; + const maxResults = req.query.maxResults + ? parseInt(req.query.maxResults as string, 10) + : 5000; + + try { + const [containers, nextMarker] = await this.metadataStore.listContainers( + createStorageContext(ctx.requestId), + account, + prefix, + maxResults, + continuation + ); + + const filesystems = containers.map(c => ({ + name: c.name, + lastModified: c.properties.lastModified.toUTCString(), + eTag: c.properties.etag + })); + + res.status(200); + res.setHeader("x-ms-request-id", ctx.requestId); + res.setHeader("x-ms-version", BLOB_API_VERSION); + if (nextMarker) { + res.setHeader("x-ms-continuation", String(nextMarker)); + } + + res.json({ filesystems }); + } catch (error: any) { + logger.error(`FilesystemHandler.list error: ${error.message}`, ctx.requestId); + sendDfsError(res, internalError(error.message)); + } + } + + public async setProperties(req: Request, res: Response): Promise { + const ctx = getDfsContext(res); + const account = ctx.account || EMULATOR_ACCOUNT_NAME; + const filesystem = ctx.filesystem!; + const now = new Date(); + const etag = `"${now.getTime().toString(16)}"`; + + try { + const metadata = this.extractMetadata(req) || {}; + + // Parse x-ms-properties header (base64 encoded key=value pairs) + const propertiesHeader = req.headers["x-ms-properties"] as string | undefined; + if (propertiesHeader) { + const pairs = propertiesHeader.split(","); + for (const pair of pairs) { + const eqIdx = pair.indexOf("="); + if (eqIdx >= 0) { + const key = pair.substring(0, eqIdx); + const value = Buffer.from(pair.substring(eqIdx + 1), "base64").toString("utf8"); + metadata[key] = value; + } + } + } + + await this.metadataStore.setContainerMetadata( + createStorageContext(ctx.requestId), + account, + filesystem, + now, + etag, + Object.keys(metadata).length > 0 ? metadata : undefined + ); + + res.status(200); + res.setHeader("ETag", etag); + res.setHeader("Last-Modified", now.toUTCString()); + res.setHeader("x-ms-request-id", ctx.requestId); + res.setHeader("x-ms-version", BLOB_API_VERSION); + res.end(); + } catch (error: any) { + if (error.statusCode === 404) { + return sendDfsError(res, filesystemNotFound(filesystem)); + } + logger.error(`FilesystemHandler.setProperties error: ${error.message}`, ctx.requestId); + sendDfsError(res, internalError(error.message)); + } + } + + private extractMetadata(req: Request): { [key: string]: string } | undefined { + const metadata: { [key: string]: string } = {}; + let hasMetadata = false; + for (const [key, value] of Object.entries(req.headers)) { + if (key.toLowerCase().startsWith("x-ms-meta-") && value) { + const metaKey = key.substring("x-ms-meta-".length); + metadata[metaKey] = Array.isArray(value) ? value.join(",") : value; + hasMetadata = true; + } + } + return hasMetadata ? metadata : undefined; + } +} diff --git a/src/blob/dfs/handlers/PathHandler.ts b/src/blob/dfs/handlers/PathHandler.ts new file mode 100644 index 000000000..48cf44e20 --- /dev/null +++ b/src/blob/dfs/handlers/PathHandler.ts @@ -0,0 +1,1159 @@ +import { Request, Response } from "express"; + +import logger from "../../../common/Logger"; +import { OAuthLevel } from "../../../common/models"; +import IExtentStore from "../../../common/persistence/IExtentStore"; +import IBlobMetadataStore, { + BlobModel, + BlockModel +} from "../../persistence/IBlobMetadataStore"; +import { getDfsContext, IDfsContext } from "../DfsContext"; +import { + sendDfsError, + pathNotFound, + pathAlreadyExists, + filesystemNotFound, + directoryNotEmpty, + internalError, + invalidSourceOrDestination +} from "../DfsErrorFactory"; +import { + EMULATOR_ACCOUNT_NAME, + BLOB_API_VERSION +} from "../../utils/constants"; +import * as Models from "../../generated/artifacts/models"; +import { createStorageContext } from "../DfsContextFactory"; +import { checkAcl, AclPermission } from "../DfsAclEnforcer"; + +const HNS_DIRECTORY_METADATA_KEY = "hdi_isfolder"; + +export default class PathHandler { + public constructor( + private readonly metadataStore: IBlobMetadataStore, + private readonly extentStore: IExtentStore, + private readonly oauth?: OAuthLevel + ) {} + + public async create(req: Request, res: Response): Promise { + const ctx = getDfsContext(res); + const account = ctx.account || EMULATOR_ACCOUNT_NAME; + const filesystem = ctx.filesystem!; + const pathName = ctx.path!; + const resource = req.query.resource as string | undefined; + const isDirectory = resource === "directory"; + + const renameSource = req.headers["x-ms-rename-source"] as string | undefined; + if (renameSource) { + return this.renamePath(req, res); + } + + try { + const now = new Date(); + const metadata: { [key: string]: string } = {}; + if (isDirectory) { + metadata[HNS_DIRECTORY_METADATA_KEY] = "true"; + } + + // Azure returns 409 PathAlreadyExists when creating a directory that + // already exists. This is required for the SDK's CreateIfNotExistsAsync + // to correctly return null for existing directories. + if (isDirectory) { + const existing = await this.safeGetBlobProperties(account, filesystem, pathName); + if (existing && existing.metadata?.[HNS_DIRECTORY_METADATA_KEY] === "true") { + return sendDfsError(res, pathAlreadyExists(pathName)); + } + } + + // Ensure intermediate directories exist + if (pathName.includes("/")) { + await this.ensureIntermediateDirectories(account, filesystem, pathName, now); + } + + const blobModel: BlobModel = { + accountName: account, + containerName: filesystem, + name: pathName, + snapshot: "", + isCommitted: true, + properties: { + lastModified: now, + etag: `"${new Date().getTime().toString(16)}"`, + contentLength: 0, + contentType: isDirectory ? undefined : "application/octet-stream", + blobType: Models.BlobType.BlockBlob, + accessTier: Models.AccessTier.Hot, + accessTierInferred: true, + creationTime: now, + legalHold: false + }, + metadata: Object.keys(metadata).length > 0 ? metadata : undefined, + committedBlocksInOrder: [], + persistency: undefined as any + }; + + await this.metadataStore.createBlob(createStorageContext(ctx.requestId), blobModel); + + // Register in HNS hierarchy table + const parentPath = pathName.includes("/") + ? pathName.substring(0, pathName.lastIndexOf("/")) + : null; + await this.metadataStore.registerHnsPath( + createStorageContext(ctx.requestId), account, filesystem, + pathName, parentPath, isDirectory + ); + + res.status(201); + res.setHeader("ETag", blobModel.properties.etag!); + res.setHeader("Last-Modified", now.toUTCString()); + res.setHeader("x-ms-request-id", ctx.requestId); + res.setHeader("x-ms-version", BLOB_API_VERSION); + res.setHeader("Content-Length", "0"); + res.end(); + } catch (error: any) { + if (error.statusCode === 404) { + return sendDfsError(res, filesystemNotFound(filesystem)); + } + logger.error(`PathHandler.create error: ${error.message}`, ctx.requestId); + sendDfsError(res, internalError(error.message)); + } + } + + public async delete(req: Request, res: Response): Promise { + const ctx = getDfsContext(res); + const account = ctx.account || EMULATOR_ACCOUNT_NAME; + const filesystem = ctx.filesystem!; + const pathName = ctx.path!; + const recursive = req.query.recursive === "true"; + + // ACL enforcement + if (!(await this.enforceAcl(ctx, res, account, filesystem, pathName, "w"))) return; + + try { + // Check if it's a directory + const blobProps = await this.safeGetBlobProperties(account, filesystem, pathName); + if (!blobProps) { + return sendDfsError(res, pathNotFound(pathName)); + } + + const isDir = blobProps.metadata?.[HNS_DIRECTORY_METADATA_KEY] === "true"; + + if (isDir) { + // List ALL blobs under this directory prefix (recursive, no delimiter) + // to check for children. This catches blobs created via both DFS and + // Blob API, regardless of whether they're in the HNS hierarchy table. + const prefix = pathName + "/"; + const [allChildren] = await this.metadataStore.listBlobs( + createStorageContext(ctx.requestId), account, filesystem, + undefined, undefined, prefix + ); + + if (allChildren.length > 0 && !recursive) { + return sendDfsError(res, directoryNotEmpty(pathName)); + } + + if (recursive && allChildren.length > 0) { + // Delete all descendant blobs + for (const child of allChildren) { + await this.metadataStore.deleteBlob( + createStorageContext(ctx.requestId), account, filesystem, child.name, {} + ); + } + // Unregister all descendants from HNS hierarchy + await this.metadataStore.unregisterHnsPathsByPrefix( + createStorageContext(ctx.requestId), account, filesystem, prefix + ); + } + } + + const leaseConditions = this.extractLeaseConditions(req); + const modifiedConditions = this.extractModifiedAccessConditions(req); + await this.metadataStore.deleteBlob( + createStorageContext(ctx.requestId), account, filesystem, pathName, + { + leaseAccessConditions: leaseConditions, + modifiedAccessConditions: modifiedConditions + } + ); + + // Unregister from HNS hierarchy + await this.metadataStore.unregisterHnsPath( + createStorageContext(ctx.requestId), account, filesystem, pathName + ); + + res.status(200); + res.setHeader("x-ms-request-id", ctx.requestId); + res.setHeader("x-ms-version", BLOB_API_VERSION); + res.end(); + } catch (error: any) { + if (error.statusCode === 404) { + return sendDfsError(res, pathNotFound(pathName)); + } + logger.error(`PathHandler.delete error: ${error.message}`, ctx.requestId); + sendDfsError(res, internalError(error.message)); + } + } + + public async getProperties(req: Request, res: Response): Promise { + const ctx = getDfsContext(res); + const account = ctx.account || EMULATOR_ACCOUNT_NAME; + const filesystem = ctx.filesystem!; + const pathName = ctx.path!; + const action = req.query.action as string | undefined; + + // ACL enforcement + if (!(await this.enforceAcl(ctx, res, account, filesystem, pathName, "r"))) return; + + try { + const leaseConditions = this.extractLeaseConditions(req); + const modifiedConditions = this.extractModifiedAccessConditions(req); + const result = await this.metadataStore.getBlobProperties( + createStorageContext(ctx.requestId), account, filesystem, pathName, + undefined, leaseConditions, modifiedConditions + ); + + const isDir = result.metadata?.[HNS_DIRECTORY_METADATA_KEY] === "true"; + + res.status(200); + res.setHeader("ETag", result.properties.etag!); + res.setHeader("Last-Modified", result.properties.lastModified.toUTCString()); + res.setHeader("x-ms-request-id", ctx.requestId); + res.setHeader("x-ms-version", BLOB_API_VERSION); + res.setHeader("x-ms-resource-type", isDir ? "directory" : "file"); + + if (!isDir) { + res.setHeader("Content-Length", String(result.properties.contentLength || 0)); + if (result.properties.contentType) { + res.setHeader("Content-Type", result.properties.contentType); + } + } else { + res.setHeader("Content-Length", "0"); + } + + // ACL headers + if (action === "getAccessControl") { + res.setHeader("x-ms-owner", (result.metadata as any)?.dfsAclOwner || "$superuser"); + res.setHeader("x-ms-group", (result.metadata as any)?.dfsAclGroup || "$superuser"); + res.setHeader("x-ms-permissions", (result.metadata as any)?.dfsAclPermissions || "rwxr-x---"); + if ((result.metadata as any)?.dfsAcl) { + res.setHeader("x-ms-acl", (result.metadata as any).dfsAcl); + } + } + + res.end(); + } catch (error: any) { + if (error.statusCode === 404) { + return sendDfsError(res, pathNotFound(pathName)); + } + logger.error(`PathHandler.getProperties error: ${error.message}`, ctx.requestId); + sendDfsError(res, internalError(error.message)); + } + } + + public async read(req: Request, res: Response): Promise { + const ctx = getDfsContext(res); + const account = ctx.account || EMULATOR_ACCOUNT_NAME; + const filesystem = ctx.filesystem!; + const pathName = ctx.path!; + + // ACL enforcement + if (!(await this.enforceAcl(ctx, res, account, filesystem, pathName, "r"))) return; + + try { + const leaseConditions = this.extractLeaseConditions(req); + const modifiedConditions = this.extractModifiedAccessConditions(req); + const blob = await this.metadataStore.downloadBlob( + createStorageContext(ctx.requestId), account, filesystem, pathName, + undefined, leaseConditions, modifiedConditions + ); + + res.status(200); + res.setHeader("ETag", blob.properties.etag!); + res.setHeader("Last-Modified", blob.properties.lastModified.toUTCString()); + res.setHeader("x-ms-request-id", ctx.requestId); + res.setHeader("x-ms-version", BLOB_API_VERSION); + res.setHeader("x-ms-resource-type", "file"); + res.setHeader("Content-Length", String(blob.properties.contentLength || 0)); + + if (blob.properties.contentType) { + res.setHeader("Content-Type", blob.properties.contentType); + } + + const hasCommittedBlocks = blob.committedBlocksInOrder && blob.committedBlocksInOrder.length > 0; + if (blob.properties.contentLength === 0 && !hasCommittedBlocks) { + return res.end(); + } + + // Read from extent store + if (hasCommittedBlocks) { + // Multi-block blob: read each block in order + for (const block of blob.committedBlocksInOrder!) { + const stream = await this.extentStore.readExtent(block.persistency); + await new Promise((resolve, reject) => { + stream.on("data", (chunk: Buffer) => res.write(chunk)); + stream.on("end", resolve); + stream.on("error", reject); + }); + } + res.end(); + } else if (blob.persistency) { + const stream = await this.extentStore.readExtent(blob.persistency); + await new Promise((resolve, reject) => { + stream.on("end", () => { res.end(); resolve(); }); + stream.on("error", reject); + stream.pipe(res); + }); + } else { + res.end(); + } + } catch (error: any) { + if (error.statusCode === 404) { + return sendDfsError(res, pathNotFound(pathName)); + } + logger.error(`PathHandler.read error: ${error.message}`, ctx.requestId); + sendDfsError(res, internalError(error.message)); + } + } + + public async listPaths(req: Request, res: Response): Promise { + const ctx = getDfsContext(res); + const account = ctx.account || EMULATOR_ACCOUNT_NAME; + const filesystem = ctx.filesystem!; + const directory = req.query.directory as string | undefined; + const recursive = req.query.recursive === "true"; + const maxResults = req.query.maxResults + ? parseInt(req.query.maxResults as string, 10) + : 5000; + const continuation = req.query.continuation as string | undefined; + + const prefix = directory ? (directory.endsWith("/") ? directory : directory + "/") : ""; + const delimiter = recursive ? undefined : "/"; + + try { + const [blobs, prefixes, nextMarker] = await this.metadataStore.listBlobs( + createStorageContext(ctx.requestId), account, filesystem, delimiter, undefined, + prefix, maxResults, continuation + ); + + const paths: any[] = []; + + for (const blob of blobs) { + // Skip the directory marker itself if it matches prefix exactly + if (blob.name === directory) continue; + + const isDir = blob.metadata?.[HNS_DIRECTORY_METADATA_KEY] === "true"; + paths.push({ + name: blob.name, + isDirectory: isDir || false, + lastModified: blob.properties.lastModified.toUTCString(), + eTag: blob.properties.etag, + contentLength: isDir ? 0 : (blob.properties.contentLength || 0), + owner: "$superuser", + group: "$superuser", + permissions: "rwxr-x---" + }); + } + + // Add prefixes as directories (for non-recursive listing) + if (prefixes) { + for (const p of prefixes) { + paths.push({ + name: p.name.endsWith("/") ? p.name.slice(0, -1) : p.name, + isDirectory: true, + lastModified: new Date().toUTCString(), + contentLength: 0, + owner: "$superuser", + group: "$superuser", + permissions: "rwxr-x---" + }); + } + } + + res.status(200); + res.setHeader("x-ms-request-id", ctx.requestId); + res.setHeader("x-ms-version", BLOB_API_VERSION); + if (nextMarker) { + res.setHeader("x-ms-continuation", nextMarker); + } + + res.json({ paths }); + } catch (error: any) { + if (error.statusCode === 404) { + return sendDfsError(res, filesystemNotFound(filesystem)); + } + logger.error(`PathHandler.listPaths error: ${error.message}`, ctx.requestId); + sendDfsError(res, internalError(error.message)); + } + } + + public async update(req: Request, res: Response): Promise { + const ctx = getDfsContext(res); + const account = ctx.account || EMULATOR_ACCOUNT_NAME; + const filesystem = ctx.filesystem!; + const pathName = ctx.path!; + + // ACL enforcement for update operations + if (!(await this.enforceAcl(ctx, res, account, filesystem, pathName, "w"))) return; + + const action = req.query.action as string; + switch (action) { + case "append": + return this.appendData(req, res); + case "flush": + return this.flushData(req, res); + case "setAccessControl": + return this.setAccessControl(req, res); + case "setAccessControlRecursive": + return this.setAccessControlRecursive(req, res); + case "setProperties": + return this.setProperties(req, res); + default: + return sendDfsError(res, { + statusCode: 400, + code: "InvalidQueryParameterValue", + message: `Value for one of the query parameters specified in the request URI is invalid. QueryParameterName: action, QueryParameterValue: ${action}` + }); + } + } + + private async appendData(req: Request, res: Response): Promise { + const ctx = getDfsContext(res); + const account = ctx.account || EMULATOR_ACCOUNT_NAME; + const filesystem = ctx.filesystem!; + const pathName = ctx.path!; + const positionParam = Array.isArray(req.query.position) + ? req.query.position[0] + : req.query.position; + const position = parseInt(String(positionParam || "0"), 10); + + try { + const rawBody = Array.isArray(req.body) ? Buffer.from(req.body) : req.body; + const body = Buffer.isBuffer(rawBody) ? rawBody : Buffer.from(rawBody || ""); + + // Content-MD5 validation + const contentMD5 = req.headers["content-md5"] as string | undefined; + if (contentMD5) { + const crypto = require("crypto"); + const computedMD5 = crypto.createHash("md5").update(body).digest("base64"); + if (computedMD5 !== contentMD5) { + return sendDfsError(res, { + statusCode: 400, + code: "Md5Mismatch", + message: "The MD5 value specified in the request did not match with the MD5 value calculated by the server." + }); + } + } + + if (body.length === 0) { + res.status(202); + res.setHeader("x-ms-request-id", ctx.requestId); + res.setHeader("x-ms-version", BLOB_API_VERSION); + return res.end(); + } + + // Write to extent store + const extentChunk = await this.extentStore.appendExtent(body); + + // Stage as an uncommitted block (reusing block blob infrastructure) + const blockId = Buffer.from( + `dfs-${position.toString().padStart(20, "0")}` + ).toString("base64"); + + const block: BlockModel = { + accountName: account, + containerName: filesystem, + blobName: pathName, + isCommitted: false, + name: blockId, + size: body.length, + persistency: extentChunk + }; + + await this.metadataStore.stageBlock( + createStorageContext(ctx.requestId), block, undefined + ); + + res.status(202); + res.setHeader("x-ms-request-id", ctx.requestId); + res.setHeader("x-ms-version", BLOB_API_VERSION); + res.setHeader("x-ms-content-length", String(body.length)); + res.end(); + } catch (error: any) { + if (error.statusCode === 404) { + return sendDfsError(res, pathNotFound(pathName)); + } + logger.error(`PathHandler.appendData error: ${error.message}`, ctx.requestId); + sendDfsError(res, internalError(error.message)); + } + } + + private async flushData(req: Request, res: Response): Promise { + const ctx = getDfsContext(res); + const account = ctx.account || EMULATOR_ACCOUNT_NAME; + const filesystem = ctx.filesystem!; + const pathName = ctx.path!; + const flushPositionParam = Array.isArray(req.query.position) + ? req.query.position[0] + : req.query.position; + const position = parseInt(String(flushPositionParam || "0"), 10); + + try { + // Get current blob to find uncommitted blocks + const blob = await this.metadataStore.downloadBlob( + createStorageContext(ctx.requestId), account, filesystem, pathName, undefined + ); + + // Get uncommitted blocks + const blockList = await this.metadataStore.getBlockList( + createStorageContext(ctx.requestId), account, filesystem, pathName, + undefined, undefined, undefined, undefined + ); + + if (!blockList.uncommittedBlocks || blockList.uncommittedBlocks.length === 0) { + // Nothing to flush — just update the blob + res.status(200); + res.setHeader("ETag", blob.properties.etag!); + res.setHeader("Last-Modified", blob.properties.lastModified.toUTCString()); + res.setHeader("x-ms-request-id", ctx.requestId); + res.setHeader("x-ms-version", BLOB_API_VERSION); + return res.end(); + } + + // Build commit block list from uncommitted blocks + const commitList = blockList.uncommittedBlocks.map(b => ({ + blockName: b.name, + blockCommitType: "Uncommitted" + })); + + const now = new Date(); + const etag = `"${now.getTime().toString(16)}"`; + + const updatedBlob: BlobModel = { + ...blob, + properties: { + ...blob.properties, + lastModified: now, + etag, + contentLength: position, + contentType: blob.properties.contentType || "application/octet-stream" + } + }; + + await this.metadataStore.commitBlockList( + createStorageContext(ctx.requestId), updatedBlob, commitList + ); + + res.status(200); + res.setHeader("ETag", etag); + res.setHeader("Last-Modified", now.toUTCString()); + res.setHeader("x-ms-request-id", ctx.requestId); + res.setHeader("x-ms-version", BLOB_API_VERSION); + res.setHeader("x-ms-resource-type", "file"); + res.setHeader("Content-Length", "0"); + res.end(); + } catch (error: any) { + if (error.statusCode === 404) { + return sendDfsError(res, pathNotFound(pathName)); + } + logger.error(`PathHandler.flushData error: ${error.message}`, ctx.requestId); + sendDfsError(res, internalError(error.message)); + } + } + + private async setAccessControl(req: Request, res: Response): Promise { + const ctx = getDfsContext(res); + const account = ctx.account || EMULATOR_ACCOUNT_NAME; + const filesystem = ctx.filesystem!; + const pathName = ctx.path!; + + try { + const result = await this.metadataStore.getBlobProperties( + createStorageContext(ctx.requestId), account, filesystem, pathName, undefined, undefined + ); + + // Store ACL info in metadata + const metadata = { ...(result.metadata || {}) }; + const owner = req.headers["x-ms-owner"] as string | undefined; + const group = req.headers["x-ms-group"] as string | undefined; + const permissions = req.headers["x-ms-permissions"] as string | undefined; + const acl = req.headers["x-ms-acl"] as string | undefined; + + if (owner) metadata["dfsAclOwner"] = owner; + if (group) metadata["dfsAclGroup"] = group; + if (permissions) metadata["dfsAclPermissions"] = permissions; + if (acl) metadata["dfsAcl"] = acl; + + const now = new Date(); + const etag = `"${now.getTime().toString(16)}"`; + + await this.metadataStore.setBlobMetadata( + createStorageContext(ctx.requestId), account, filesystem, pathName, + undefined, metadata + ); + + res.status(200); + res.setHeader("ETag", etag); + res.setHeader("Last-Modified", now.toUTCString()); + res.setHeader("x-ms-request-id", ctx.requestId); + res.setHeader("x-ms-version", BLOB_API_VERSION); + res.end(); + } catch (error: any) { + if (error.statusCode === 404) { + return sendDfsError(res, pathNotFound(pathName)); + } + logger.error(`PathHandler.setAccessControl error: ${error.message}`, ctx.requestId); + sendDfsError(res, internalError(error.message)); + } + } + + private async setAccessControlRecursive(req: Request, res: Response): Promise { + const ctx = getDfsContext(res); + const account = ctx.account || EMULATOR_ACCOUNT_NAME; + const filesystem = ctx.filesystem!; + const pathName = ctx.path!; + const mode = req.query.mode as string || "set"; // set, modify, remove + const acl = req.headers["x-ms-acl"] as string | undefined; + const maxRecords = req.query.maxRecords + ? parseInt(req.query.maxRecords as string, 10) + : 2000; + const continuation = req.query.continuation as string | undefined; + + try { + const prefix = pathName.endsWith("/") ? pathName : pathName + "/"; + + const [blobs, , nextMarker] = await this.metadataStore.listBlobs( + createStorageContext(ctx.requestId), account, filesystem, + undefined, undefined, prefix, maxRecords, continuation + ); + + let directoriesSuccessful = 0; + let filesSuccessful = 0; + const failureCount = 0; + + // Also apply to the path itself + const allPaths = [pathName, ...blobs.map(b => b.name)]; + + for (const blobPath of allPaths) { + try { + const props = await this.metadataStore.getBlobProperties( + createStorageContext(ctx.requestId), account, filesystem, + blobPath, undefined, undefined + ); + + const metadata = { ...(props.metadata || {}) }; + const isDir = metadata[HNS_DIRECTORY_METADATA_KEY] === "true"; + + if (acl) { + if (mode === "set") { + metadata["dfsAcl"] = acl; + } else if (mode === "modify") { + // Merge: new ACL entries override existing ones with same qualifier + const existing = (metadata["dfsAcl"] || "").split(",").filter(Boolean); + const incoming = acl.split(","); + const merged = new Map(); + for (const entry of existing) { + const key = entry.split(":").slice(0, 2).join(":"); + merged.set(key, entry); + } + for (const entry of incoming) { + const key = entry.split(":").slice(0, 2).join(":"); + merged.set(key, entry); + } + metadata["dfsAcl"] = Array.from(merged.values()).join(","); + } else if (mode === "remove") { + // Remove specified ACL entries + const existing = (metadata["dfsAcl"] || "").split(",").filter(Boolean); + const toRemove = new Set(acl.split(",").map((e: string) => e.split(":").slice(0, 2).join(":"))); + metadata["dfsAcl"] = existing + .filter((e: string) => !toRemove.has(e.split(":").slice(0, 2).join(":"))) + .join(","); + } + } + + await this.metadataStore.setBlobMetadata( + createStorageContext(ctx.requestId), account, filesystem, + blobPath, undefined, metadata + ); + + if (isDir) { + directoriesSuccessful++; + } else { + filesSuccessful++; + } + } catch { + // Skip failures for individual paths + } + } + + res.status(200); + res.setHeader("x-ms-request-id", ctx.requestId); + res.setHeader("x-ms-version", BLOB_API_VERSION); + if (nextMarker) { + res.setHeader("x-ms-continuation", nextMarker); + } + + res.json({ + directoriesSuccessful, + filesSuccessful, + failureCount + }); + } catch (error: any) { + if (error.statusCode === 404) { + return sendDfsError(res, pathNotFound(pathName)); + } + logger.error(`PathHandler.setAccessControlRecursive error: ${error.message}`, ctx.requestId); + sendDfsError(res, internalError(error.message)); + } + } + + private async setProperties(req: Request, res: Response): Promise { + const ctx = getDfsContext(res); + const account = ctx.account || EMULATOR_ACCOUNT_NAME; + const filesystem = ctx.filesystem!; + const pathName = ctx.path!; + + try { + const result = await this.metadataStore.getBlobProperties( + createStorageContext(ctx.requestId), account, filesystem, pathName, undefined, undefined + ); + + const metadata = { ...(result.metadata || {}) }; + + // Parse x-ms-properties header (base64 encoded key=value pairs) + const propertiesHeader = req.headers["x-ms-properties"] as string | undefined; + if (propertiesHeader) { + const pairs = propertiesHeader.split(","); + for (const pair of pairs) { + const eqIdx = pair.indexOf("="); + if (eqIdx >= 0) { + const key = pair.substring(0, eqIdx); + const value = Buffer.from(pair.substring(eqIdx + 1), "base64").toString("utf8"); + metadata[key] = value; + } + } + } + + const now = new Date(); + const etag = `"${now.getTime().toString(16)}"`; + + await this.metadataStore.setBlobMetadata( + createStorageContext(ctx.requestId), account, filesystem, pathName, + undefined, metadata + ); + + res.status(200); + res.setHeader("ETag", etag); + res.setHeader("Last-Modified", now.toUTCString()); + res.setHeader("x-ms-request-id", ctx.requestId); + res.setHeader("x-ms-version", BLOB_API_VERSION); + res.end(); + } catch (error: any) { + if (error.statusCode === 404) { + return sendDfsError(res, pathNotFound(pathName)); + } + logger.error(`PathHandler.setProperties error: ${error.message}`, ctx.requestId); + sendDfsError(res, internalError(error.message)); + } + } + + public async lease(req: Request, res: Response): Promise { + const leaseAction = (req.headers["x-ms-lease-action"] as string || "").toLowerCase(); + switch (leaseAction) { + case "acquire": + return this.acquireLease(req, res); + case "release": + return this.releaseLease(req, res); + case "renew": + return this.renewLease(req, res); + case "break": + return this.breakLease(req, res); + case "change": + return this.changeLease(req, res); + default: + return sendDfsError(res, { + statusCode: 400, + code: "InvalidHeaderValue", + message: `The value for one of the HTTP headers is not in the correct format. Header: x-ms-lease-action, Value: ${leaseAction}` + }); + } + } + + private async acquireLease(req: Request, res: Response): Promise { + const ctx = getDfsContext(res); + const account = ctx.account || EMULATOR_ACCOUNT_NAME; + const filesystem = ctx.filesystem!; + const pathName = ctx.path!; + + try { + const duration = parseInt(req.headers["x-ms-lease-duration"] as string || "-1", 10); + const proposedLeaseId = req.headers["x-ms-proposed-lease-id"] as string | undefined; + const modifiedConditions = this.extractModifiedAccessConditions(req); + + const result = await this.metadataStore.acquireBlobLease( + createStorageContext(ctx.requestId), + account, filesystem, pathName, duration, proposedLeaseId, + { modifiedAccessConditions: modifiedConditions } + ); + + res.status(201); + res.setHeader("ETag", result.properties.etag!); + res.setHeader("Last-Modified", result.properties.lastModified.toUTCString()); + res.setHeader("x-ms-lease-id", result.leaseId!); + res.setHeader("x-ms-request-id", ctx.requestId); + res.setHeader("x-ms-version", BLOB_API_VERSION); + res.end(); + } catch (error: any) { + this.handleLeaseError(res, error, ctx.requestId, pathName); + } + } + + private async releaseLease(req: Request, res: Response): Promise { + const ctx = getDfsContext(res); + const account = ctx.account || EMULATOR_ACCOUNT_NAME; + const filesystem = ctx.filesystem!; + const pathName = ctx.path!; + + try { + const leaseId = req.headers["x-ms-lease-id"] as string; + const modifiedConditions = this.extractModifiedAccessConditions(req); + + await this.metadataStore.releaseBlobLease( + createStorageContext(ctx.requestId), + account, filesystem, pathName, leaseId, + { modifiedAccessConditions: modifiedConditions } + ); + + res.status(200); + res.setHeader("x-ms-request-id", ctx.requestId); + res.setHeader("x-ms-version", BLOB_API_VERSION); + res.end(); + } catch (error: any) { + this.handleLeaseError(res, error, ctx.requestId, pathName); + } + } + + private async renewLease(req: Request, res: Response): Promise { + const ctx = getDfsContext(res); + const account = ctx.account || EMULATOR_ACCOUNT_NAME; + const filesystem = ctx.filesystem!; + const pathName = ctx.path!; + + try { + const leaseId = req.headers["x-ms-lease-id"] as string; + const modifiedConditions = this.extractModifiedAccessConditions(req); + + const result = await this.metadataStore.renewBlobLease( + createStorageContext(ctx.requestId), + account, filesystem, pathName, leaseId, + { modifiedAccessConditions: modifiedConditions } + ); + + res.status(200); + res.setHeader("ETag", result.properties.etag!); + res.setHeader("Last-Modified", result.properties.lastModified.toUTCString()); + res.setHeader("x-ms-lease-id", result.leaseId!); + res.setHeader("x-ms-request-id", ctx.requestId); + res.setHeader("x-ms-version", BLOB_API_VERSION); + res.end(); + } catch (error: any) { + this.handleLeaseError(res, error, ctx.requestId, pathName); + } + } + + private async breakLease(req: Request, res: Response): Promise { + const ctx = getDfsContext(res); + const account = ctx.account || EMULATOR_ACCOUNT_NAME; + const filesystem = ctx.filesystem!; + const pathName = ctx.path!; + + try { + const breakPeriod = req.headers["x-ms-lease-break-period"] + ? parseInt(req.headers["x-ms-lease-break-period"] as string, 10) + : undefined; + const modifiedConditions = this.extractModifiedAccessConditions(req); + + const result = await this.metadataStore.breakBlobLease( + createStorageContext(ctx.requestId), + account, filesystem, pathName, breakPeriod, + { modifiedAccessConditions: modifiedConditions } + ); + + res.status(202); + res.setHeader("ETag", result.properties.etag!); + res.setHeader("Last-Modified", result.properties.lastModified.toUTCString()); + if (result.leaseTime !== undefined) { + res.setHeader("x-ms-lease-time", String(result.leaseTime)); + } + res.setHeader("x-ms-request-id", ctx.requestId); + res.setHeader("x-ms-version", BLOB_API_VERSION); + res.end(); + } catch (error: any) { + this.handleLeaseError(res, error, ctx.requestId, pathName); + } + } + + private async changeLease(req: Request, res: Response): Promise { + const ctx = getDfsContext(res); + const account = ctx.account || EMULATOR_ACCOUNT_NAME; + const filesystem = ctx.filesystem!; + const pathName = ctx.path!; + + try { + const leaseId = req.headers["x-ms-lease-id"] as string; + const proposedLeaseId = req.headers["x-ms-proposed-lease-id"] as string; + const modifiedConditions = this.extractModifiedAccessConditions(req); + + const result = await this.metadataStore.changeBlobLease( + createStorageContext(ctx.requestId), + account, filesystem, pathName, leaseId, proposedLeaseId, + { modifiedAccessConditions: modifiedConditions } + ); + + res.status(200); + res.setHeader("ETag", result.properties.etag!); + res.setHeader("Last-Modified", result.properties.lastModified.toUTCString()); + res.setHeader("x-ms-lease-id", result.leaseId!); + res.setHeader("x-ms-request-id", ctx.requestId); + res.setHeader("x-ms-version", BLOB_API_VERSION); + res.end(); + } catch (error: any) { + this.handleLeaseError(res, error, ctx.requestId, pathName); + } + } + + private handleLeaseError(res: Response, error: any, requestId: string, pathName: string): void { + if (error.statusCode === 404) { + return sendDfsError(res, pathNotFound(pathName)); + } + if (error.statusCode === 409 || error.statusCode === 412) { + return sendDfsError(res, { + statusCode: error.statusCode, + code: error.storageErrorCode || error.code || "LeaseOperationFailed", + message: error.storageErrorMessage || error.message + }); + } + logger.error(`PathHandler.lease error: ${error.message}`, requestId); + sendDfsError(res, internalError(error.message)); + } + + private async renamePath(req: Request, res: Response): Promise { + const ctx = getDfsContext(res); + const account = ctx.account || EMULATOR_ACCOUNT_NAME; + const destFilesystem = ctx.filesystem!; + const destPath = ctx.path!; + const renameSource = req.headers["x-ms-rename-source"] as string; + + try { + // Parse rename source: /{filesystem}/{path}?sastoken + const sourceUrl = new URL(renameSource, "http://localhost"); + const sourceParts = sourceUrl.pathname.split("/").filter(p => p); + + // Handle both /{account}/{filesystem}/{path} and /{filesystem}/{path} + let sourceFilesystem: string; + let sourcePath: string; + if (sourceParts.length >= 3 && sourceParts[0] === account) { + sourceFilesystem = sourceParts[1]; + sourcePath = sourceParts.slice(2).join("/"); + } else if (sourceParts.length >= 2) { + sourceFilesystem = sourceParts[0]; + sourcePath = sourceParts.slice(1).join("/"); + } else { + return sendDfsError(res, invalidSourceOrDestination( + `Invalid rename source: ${renameSource}` + )); + } + + // Get source blob to check if it exists and whether it's a directory + const sourceBlob = await this.safeGetBlobProperties(account, sourceFilesystem, sourcePath); + if (!sourceBlob) { + return sendDfsError(res, pathNotFound(sourcePath)); + } + + const isDir = sourceBlob.metadata?.[HNS_DIRECTORY_METADATA_KEY] === "true"; + + if (isDir) { + // Atomically rename all children by prefix + await this.metadataStore.renameBlobsByPrefix( + createStorageContext(ctx.requestId), + account, + sourceFilesystem, + sourcePath + "/", + destFilesystem, + destPath + "/" + ); + } + + // Atomically rename the path itself (file or directory marker) + const result = await this.metadataStore.renameBlob( + createStorageContext(ctx.requestId), + account, + sourceFilesystem, + sourcePath, + destFilesystem, + destPath + ); + + const now = new Date(); + + // Update HNS hierarchy for the renamed paths + await this.metadataStore.renameHnsPaths( + createStorageContext(ctx.requestId), + account, sourceFilesystem, sourcePath, + destFilesystem, destPath + ); + + // Ensure intermediate directories for destination + if (destPath.includes("/")) { + await this.ensureIntermediateDirectories(account, destFilesystem, destPath, now); + } + + res.status(201); + res.setHeader("ETag", result.etag!); + res.setHeader("Last-Modified", result.lastModified!.toUTCString()); + res.setHeader("x-ms-request-id", ctx.requestId); + res.setHeader("x-ms-version", BLOB_API_VERSION); + res.setHeader("Content-Length", "0"); + res.end(); + } catch (error: any) { + if (error.statusCode === 404) { + return sendDfsError(res, pathNotFound(renameSource)); + } + logger.error(`PathHandler.renamePath error: ${error.message}`, ctx.requestId); + sendDfsError(res, internalError(error.message)); + } + } + + private async ensureIntermediateDirectories( + account: string, + filesystem: string, + pathName: string, + now: Date + ): Promise { + const parts = pathName.split("/"); + // Skip the last part (the file/dir being created) + for (let i = 1; i < parts.length; i++) { + const dirPath = parts.slice(0, i).join("/"); + const existing = await this.safeGetBlobProperties(account, filesystem, dirPath); + if (!existing) { + const dirBlob: BlobModel = { + accountName: account, + containerName: filesystem, + name: dirPath, + snapshot: "", + isCommitted: true, + properties: { + lastModified: now, + etag: `"${now.getTime().toString(16)}-${i}"`, + contentLength: 0, + blobType: Models.BlobType.BlockBlob, + accessTier: Models.AccessTier.Hot, + accessTierInferred: true, + creationTime: now, + legalHold: false + }, + metadata: { [HNS_DIRECTORY_METADATA_KEY]: "true" }, + committedBlocksInOrder: [], + persistency: undefined as any + }; + try { + await this.metadataStore.createBlob(createStorageContext(), dirBlob); + // Register intermediate directory in HNS hierarchy + const parentDir = i > 1 ? parts.slice(0, i - 1).join("/") : null; + await this.metadataStore.registerHnsPath( + createStorageContext(), account, filesystem, + dirPath, parentDir, true + ); + } catch { + // Ignore if already exists (race condition) + } + } + } + } + + /** + * Enforce ACL on a path operation when --oauth acl is enabled. + * Returns true if allowed, sends error response and returns false if denied. + */ + private async enforceAcl( + ctx: IDfsContext, + res: Response, + account: string, + filesystem: string, + pathName: string, + requiredPermission: AclPermission + ): Promise { + if (this.oauth !== OAuthLevel.ACL || !ctx.identity) { + return true; // ACL enforcement not active + } + + try { + const blobProps = await this.safeGetBlobProperties(account, filesystem, pathName); + if (!blobProps) { + return true; // Path doesn't exist yet (create) — allow + } + + const owner = blobProps.metadata?.dfsAclOwner; + const group = blobProps.metadata?.dfsAclGroup; + const permissions = blobProps.metadata?.dfsAclPermissions; + const acl = blobProps.metadata?.dfsAcl; + + const result = checkAcl(ctx.identity, owner, group, permissions, acl, requiredPermission); + + if (!result.allowed) { + logger.info( + `PathHandler ACL denied: ${result.reason} (path=${pathName}, perm=${requiredPermission})`, + ctx.requestId + ); + sendDfsError(res, { + statusCode: 403, + code: "AuthorizationPermissionMismatch", + message: `This request is not authorized to perform this operation using this permission. Required: ${requiredPermission}` + }); + return false; + } + + return true; + } catch { + return true; // On error, allow through (best-effort enforcement) + } + } + + private extractLeaseConditions(req: Request): Models.LeaseAccessConditions | undefined { + const leaseId = req.headers["x-ms-lease-id"] as string | undefined; + if (leaseId) { + return { leaseId }; + } + return undefined; + } + + private extractModifiedAccessConditions(req: Request): Models.ModifiedAccessConditions | undefined { + const ifMatch = req.headers["if-match"] as string | undefined; + const ifNoneMatch = req.headers["if-none-match"] as string | undefined; + const ifModifiedSince = req.headers["if-modified-since"] as string | undefined; + const ifUnmodifiedSince = req.headers["if-unmodified-since"] as string | undefined; + + if (!ifMatch && !ifNoneMatch && !ifModifiedSince && !ifUnmodifiedSince) { + return undefined; + } + + return { + ifMatch, + ifNoneMatch, + ifModifiedSince: ifModifiedSince ? new Date(ifModifiedSince) : undefined, + ifUnmodifiedSince: ifUnmodifiedSince ? new Date(ifUnmodifiedSince) : undefined + }; + } + + private async safeGetBlobProperties( + account: string, + filesystem: string, + pathName: string + ): Promise { + try { + return await this.metadataStore.getBlobProperties( + createStorageContext(), account, filesystem, pathName, undefined, undefined + ); + } catch { + return undefined; + } + } +} diff --git a/src/blob/generated-dfs/Context.ts b/src/blob/generated-dfs/Context.ts new file mode 100644 index 000000000..76cef2f28 --- /dev/null +++ b/src/blob/generated-dfs/Context.ts @@ -0,0 +1,42 @@ +/* + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + * + * DFS Context — wraps the IDfsContext from the DFS middleware layer + * and provides a typed interface compatible with the generated handler pattern. + */ + +import { DfsOperation } from "./artifacts/operation"; + +export interface IDfsRequestContext { + requestId: string; + startTime: Date; + account?: string; + filesystem?: string; + path?: string; + isSecondary?: boolean; + operation?: DfsOperation; + authenticationPath?: string; +} + +/** + * Context object passed to generated DFS handler methods. + * Mirrors the pattern of src/blob/generated/Context.ts but tailored for DFS. + */ +export default class Context { + public readonly contextId: string; + public readonly startTime: Date; + public readonly account: string | undefined; + public readonly filesystem: string | undefined; + public readonly path: string | undefined; + public operation: DfsOperation | undefined; + + public constructor(dfsContext: IDfsRequestContext) { + this.contextId = dfsContext.requestId; + this.startTime = dfsContext.startTime; + this.account = dfsContext.account; + this.filesystem = dfsContext.filesystem; + this.path = dfsContext.path; + this.operation = dfsContext.operation; + } +} diff --git a/src/blob/generated-dfs/artifacts/models.ts b/src/blob/generated-dfs/artifacts/models.ts new file mode 100644 index 000000000..28cbbacf7 --- /dev/null +++ b/src/blob/generated-dfs/artifacts/models.ts @@ -0,0 +1,287 @@ +/* + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + * + * Code generated by Microsoft (R) AutoRest Code Generator. + * Changes may cause incorrect behavior and will be lost if the code is regenerated. + */ + +// --------------------------------------------------------------------------- +// Common models +// --------------------------------------------------------------------------- + +export interface ModifiedAccessConditions { + ifModifiedSince?: Date; + ifUnmodifiedSince?: Date; + ifMatch?: string; + ifNoneMatch?: string; +} + +export interface SourceModifiedAccessConditions { + sourceIfMatch?: string; + sourceIfNoneMatch?: string; + sourceIfModifiedSince?: Date; + sourceIfUnmodifiedSince?: Date; +} + +export interface LeaseAccessConditions { + leaseId?: string; +} + +// --------------------------------------------------------------------------- +// Filesystem models +// --------------------------------------------------------------------------- + +export interface FilesystemItem { + name: string; + lastModified: string; + eTag: string; +} + +export interface FilesystemListResponse { + filesystems?: FilesystemItem[]; +} + +export interface FilesystemListOptionalParams { + prefix?: string; + continuation?: string; + maxResults?: number; +} + +export interface FilesystemCreateOptionalParams { + properties?: string; +} + +export interface FilesystemCreateResponse { + statusCode: 201; + eTag?: string; + lastModified?: Date; + namespaceEnabled?: string; + requestId?: string; + version?: string; +} + +export interface FilesystemDeleteOptionalParams { + modifiedAccessConditions?: ModifiedAccessConditions; +} + +export interface FilesystemDeleteResponse { + statusCode: 202; + requestId?: string; + version?: string; +} + +export interface FilesystemGetPropertiesOptionalParams { + // No optional params beyond standard headers +} + +export interface FilesystemGetPropertiesResponse { + statusCode: 200; + eTag?: string; + lastModified?: Date; + properties?: string; + namespaceEnabled?: string; + requestId?: string; + version?: string; +} + +export interface FilesystemSetPropertiesOptionalParams { + properties?: string; + modifiedAccessConditions?: ModifiedAccessConditions; +} + +export interface FilesystemSetPropertiesResponse { + statusCode: 200; + eTag?: string; + lastModified?: Date; + requestId?: string; + version?: string; +} + +export interface FilesystemListPathsOptionalParams { + directory?: string; + recursive?: boolean; + continuation?: string; + maxResults?: number; + upn?: boolean; +} + +// --------------------------------------------------------------------------- +// Path models +// --------------------------------------------------------------------------- + +export type PathResourceType = "file" | "directory"; + +export interface PathItem { + name: string; + isDirectory?: boolean; + lastModified: string; + eTag?: string; + contentLength?: number; + owner?: string; + group?: string; + permissions?: string; +} + +export interface PathListResponse { + paths?: PathItem[]; +} + +export interface PathCreateOptionalParams { + resource?: PathResourceType; + continuation?: string; + renameSource?: string; + properties?: string; + permissions?: string; + umask?: string; + leaseAccessConditions?: LeaseAccessConditions; + modifiedAccessConditions?: ModifiedAccessConditions; + sourceModifiedAccessConditions?: SourceModifiedAccessConditions; +} + +export interface PathCreateResponse { + statusCode: 201; + eTag?: string; + lastModified?: Date; + continuation?: string; + contentLength?: number; + requestId?: string; + version?: string; +} + +export interface PathDeleteOptionalParams { + recursive?: boolean; + continuation?: string; + leaseAccessConditions?: LeaseAccessConditions; + modifiedAccessConditions?: ModifiedAccessConditions; +} + +export interface PathDeleteResponse { + statusCode: 200; + continuation?: string; + requestId?: string; + version?: string; +} + +export interface PathGetPropertiesOptionalParams { + action?: "getAccessControl" | "getStatus"; + upn?: boolean; + leaseAccessConditions?: LeaseAccessConditions; + modifiedAccessConditions?: ModifiedAccessConditions; +} + +export interface PathGetPropertiesResponse { + statusCode: 200; + eTag?: string; + lastModified?: Date; + resourceType?: string; + properties?: string; + owner?: string; + group?: string; + permissions?: string; + acl?: string; + leaseDuration?: string; + leaseState?: string; + leaseStatus?: string; + contentLength?: number; + contentType?: string; + requestId?: string; + version?: string; +} + +export interface PathReadOptionalParams { + range?: string; + leaseAccessConditions?: LeaseAccessConditions; + modifiedAccessConditions?: ModifiedAccessConditions; +} + +export interface PathReadResponse { + statusCode: 200; + body?: NodeJS.ReadableStream; + acceptRanges?: string; + contentRange?: string; + eTag?: string; + lastModified?: Date; + resourceType?: string; + properties?: string; + leaseDuration?: string; + leaseState?: string; + leaseStatus?: string; + contentLength?: number; + contentType?: string; + requestId?: string; + version?: string; +} + +export type PathUpdateAction = + | "append" + | "flush" + | "setAccessControl" + | "setAccessControlRecursive" + | "setProperties"; + +export type AclMode = "set" | "modify" | "remove"; + +export interface PathUpdateOptionalParams { + action: PathUpdateAction; + mode?: AclMode; + position?: number; + retainUncommittedData?: boolean; + close?: boolean; + contentLength?: number; + contentMD5?: string; + properties?: string; + owner?: string; + group?: string; + permissions?: string; + acl?: string; + leaseAccessConditions?: LeaseAccessConditions; + modifiedAccessConditions?: ModifiedAccessConditions; + maxRecords?: number; + continuation?: string; +} + +export interface PathUpdateResponse { + statusCode: 200 | 202; + eTag?: string; + lastModified?: Date; + contentLength?: number; + continuation?: string; + requestId?: string; + version?: string; + // For setAccessControlRecursive + directoriesSuccessful?: number; + filesSuccessful?: number; + failureCount?: number; +} + +export type LeaseAction = "acquire" | "release" | "renew" | "break" | "change"; + +export interface PathLeaseOptionalParams { + leaseAction: LeaseAction; + leaseDuration?: number; + leaseBreakPeriod?: number; + leaseId?: string; + proposedLeaseId?: string; + modifiedAccessConditions?: ModifiedAccessConditions; +} + +export interface PathLeaseResponse { + statusCode: 200 | 201 | 202; + eTag?: string; + lastModified?: Date; + leaseId?: string; + leaseTime?: number; + requestId?: string; + version?: string; +} + +// --------------------------------------------------------------------------- +// Error model +// --------------------------------------------------------------------------- + +export interface StorageError { + statusCode: number; + code: string; + message: string; +} diff --git a/src/blob/generated-dfs/artifacts/operation.ts b/src/blob/generated-dfs/artifacts/operation.ts new file mode 100644 index 000000000..3be500c71 --- /dev/null +++ b/src/blob/generated-dfs/artifacts/operation.ts @@ -0,0 +1,23 @@ +/* + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + * + * Code generated by Microsoft (R) AutoRest Code Generator. + * Changes may cause incorrect behavior and will be lost if the code is regenerated. + */ + +export enum DfsOperation { + Filesystem_Create, + Filesystem_Delete, + Filesystem_GetProperties, + Filesystem_SetProperties, + Filesystem_List, + Filesystem_ListPaths, + Path_Create, + Path_Delete, + Path_GetProperties, + Path_Read, + Path_Update, + Path_Lease, +} +export default DfsOperation; diff --git a/src/blob/generated-dfs/artifacts/specifications.ts b/src/blob/generated-dfs/artifacts/specifications.ts new file mode 100644 index 000000000..10cb0e182 --- /dev/null +++ b/src/blob/generated-dfs/artifacts/specifications.ts @@ -0,0 +1,104 @@ +/* + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + * + * Code generated by Microsoft (R) AutoRest Code Generator. + * Changes may cause incorrect behavior and will be lost if the code is regenerated. + */ + +import { DfsOperation } from "./operation"; + +/** + * Specification for matching an incoming HTTP request to a DFS operation. + * Used by the dispatch middleware to route requests. + */ +export interface IDfsOperationSpec { + httpMethod: string; + /** If true, the request must have a filesystem but no path in the URL. */ + filesystemOnly?: boolean; + /** If true, the request must have both filesystem and path in the URL. */ + requiresPath?: boolean; + /** Query parameters that must be present with specific values. */ + queryConditions?: { [key: string]: string | string[] | true }; + /** Headers that must be present (value = true means any value). */ + headerConditions?: { [key: string]: string | string[] | true }; +} + +/** + * Dispatch specifications for all DFS operations. + * The dispatch middleware iterates these and selects the best match. + */ +const Specifications: { [key: number]: IDfsOperationSpec } = {}; + +// ---- Filesystem operations ---- + +Specifications[DfsOperation.Filesystem_List] = { + httpMethod: "GET", + filesystemOnly: false, + queryConditions: { resource: "account" } +}; + +Specifications[DfsOperation.Filesystem_Create] = { + httpMethod: "PUT", + filesystemOnly: true, + queryConditions: { resource: "filesystem" } +}; + +Specifications[DfsOperation.Filesystem_Delete] = { + httpMethod: "DELETE", + filesystemOnly: true, + queryConditions: { resource: "filesystem" } +}; + +Specifications[DfsOperation.Filesystem_GetProperties] = { + httpMethod: "HEAD", + filesystemOnly: true, + queryConditions: { resource: "filesystem" } +}; + +Specifications[DfsOperation.Filesystem_SetProperties] = { + httpMethod: "PATCH", + filesystemOnly: true, + queryConditions: { resource: "filesystem" } +}; + +Specifications[DfsOperation.Filesystem_ListPaths] = { + httpMethod: "GET", + filesystemOnly: true, + queryConditions: { resource: "filesystem" } +}; + +// ---- Path operations ---- + +Specifications[DfsOperation.Path_Create] = { + httpMethod: "PUT", + requiresPath: true +}; + +Specifications[DfsOperation.Path_Delete] = { + httpMethod: "DELETE", + requiresPath: true +}; + +Specifications[DfsOperation.Path_GetProperties] = { + httpMethod: "HEAD", + requiresPath: true +}; + +Specifications[DfsOperation.Path_Read] = { + httpMethod: "GET", + requiresPath: true +}; + +Specifications[DfsOperation.Path_Update] = { + httpMethod: "PATCH", + requiresPath: true +}; + +Specifications[DfsOperation.Path_Lease] = { + httpMethod: "POST", + requiresPath: true, + headerConditions: { "x-ms-lease-action": true } +}; + +export default Specifications; diff --git a/src/blob/generated-dfs/handlers/IFilesystemHandler.ts b/src/blob/generated-dfs/handlers/IFilesystemHandler.ts new file mode 100644 index 000000000..4e03be246 --- /dev/null +++ b/src/blob/generated-dfs/handlers/IFilesystemHandler.ts @@ -0,0 +1,42 @@ +/* + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + * + * Code generated by Microsoft (R) AutoRest Code Generator. + * Changes may cause incorrect behavior and will be lost if the code is regenerated. + */ + +import * as DfsModels from "../artifacts/models"; +import Context from "../Context"; + +export default interface IFilesystemHandler { + create( + options: DfsModels.FilesystemCreateOptionalParams, + context: Context + ): Promise; + + delete( + options: DfsModels.FilesystemDeleteOptionalParams, + context: Context + ): Promise; + + getProperties( + options: DfsModels.FilesystemGetPropertiesOptionalParams, + context: Context + ): Promise; + + setProperties( + options: DfsModels.FilesystemSetPropertiesOptionalParams, + context: Context + ): Promise; + + list( + options: DfsModels.FilesystemListOptionalParams, + context: Context + ): Promise; + + listPaths( + options: DfsModels.FilesystemListPathsOptionalParams, + context: Context + ): Promise; +} diff --git a/src/blob/generated-dfs/handlers/IHandlers.ts b/src/blob/generated-dfs/handlers/IHandlers.ts new file mode 100644 index 000000000..6baf3a671 --- /dev/null +++ b/src/blob/generated-dfs/handlers/IHandlers.ts @@ -0,0 +1,16 @@ +/* + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + * + * Code generated by Microsoft (R) AutoRest Code Generator. + * Changes may cause incorrect behavior and will be lost if the code is regenerated. + */ + +import IFilesystemHandler from "./IFilesystemHandler"; +import IPathHandler from "./IPathHandler"; + +export interface IDfsHandlers { + filesystemHandler: IFilesystemHandler; + pathHandler: IPathHandler; +} +export default IDfsHandlers; diff --git a/src/blob/generated-dfs/handlers/IPathHandler.ts b/src/blob/generated-dfs/handlers/IPathHandler.ts new file mode 100644 index 000000000..653b9efb5 --- /dev/null +++ b/src/blob/generated-dfs/handlers/IPathHandler.ts @@ -0,0 +1,44 @@ +/* + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + * + * Code generated by Microsoft (R) AutoRest Code Generator. + * Changes may cause incorrect behavior and will be lost if the code is regenerated. + */ + +import * as DfsModels from "../artifacts/models"; +import Context from "../Context"; + +export default interface IPathHandler { + create( + body: NodeJS.ReadableStream | undefined, + options: DfsModels.PathCreateOptionalParams, + context: Context + ): Promise; + + delete( + options: DfsModels.PathDeleteOptionalParams, + context: Context + ): Promise; + + getProperties( + options: DfsModels.PathGetPropertiesOptionalParams, + context: Context + ): Promise; + + read( + options: DfsModels.PathReadOptionalParams, + context: Context + ): Promise; + + update( + body: NodeJS.ReadableStream | undefined, + options: DfsModels.PathUpdateOptionalParams, + context: Context + ): Promise; + + lease( + options: DfsModels.PathLeaseOptionalParams, + context: Context + ): Promise; +} diff --git a/src/blob/generated-dfs/handlers/handlerMappers.ts b/src/blob/generated-dfs/handlers/handlerMappers.ts new file mode 100644 index 000000000..13783c249 --- /dev/null +++ b/src/blob/generated-dfs/handlers/handlerMappers.ts @@ -0,0 +1,86 @@ +/* + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + * + * Code generated by Microsoft (R) AutoRest Code Generator. + * Changes may cause incorrect behavior and will be lost if the code is regenerated. + */ + +import { DfsOperation } from "../artifacts/operation"; + +export interface IHandlerPath { + handler: string; + method: string; + arguments: string[]; +} + +const operationHandlerMapping: { [key: number]: IHandlerPath } = {}; + +operationHandlerMapping[DfsOperation.Filesystem_Create] = { + arguments: ["options"], + handler: "filesystemHandler", + method: "create" +}; +operationHandlerMapping[DfsOperation.Filesystem_Delete] = { + arguments: ["options"], + handler: "filesystemHandler", + method: "delete" +}; +operationHandlerMapping[DfsOperation.Filesystem_GetProperties] = { + arguments: ["options"], + handler: "filesystemHandler", + method: "getProperties" +}; +operationHandlerMapping[DfsOperation.Filesystem_SetProperties] = { + arguments: ["options"], + handler: "filesystemHandler", + method: "setProperties" +}; +operationHandlerMapping[DfsOperation.Filesystem_List] = { + arguments: ["options"], + handler: "filesystemHandler", + method: "list" +}; +operationHandlerMapping[DfsOperation.Filesystem_ListPaths] = { + arguments: ["options"], + handler: "filesystemHandler", + method: "listPaths" +}; +operationHandlerMapping[DfsOperation.Path_Create] = { + arguments: ["body", "options"], + handler: "pathHandler", + method: "create" +}; +operationHandlerMapping[DfsOperation.Path_Delete] = { + arguments: ["options"], + handler: "pathHandler", + method: "delete" +}; +operationHandlerMapping[DfsOperation.Path_GetProperties] = { + arguments: ["options"], + handler: "pathHandler", + method: "getProperties" +}; +operationHandlerMapping[DfsOperation.Path_Read] = { + arguments: ["options"], + handler: "pathHandler", + method: "read" +}; +operationHandlerMapping[DfsOperation.Path_Update] = { + arguments: ["body", "options"], + handler: "pathHandler", + method: "update" +}; +operationHandlerMapping[DfsOperation.Path_Lease] = { + arguments: ["options"], + handler: "pathHandler", + method: "lease" +}; + +export function getHandlerByOperation( + operation: DfsOperation +): IHandlerPath | undefined { + return operationHandlerMapping[operation]; +} + +export default operationHandlerMapping; diff --git a/src/blob/main.ts b/src/blob/main.ts index 43d4d3ce8..e8191036d 100644 --- a/src/blob/main.ts +++ b/src/blob/main.ts @@ -6,18 +6,27 @@ import BlobServer from "./BlobServer"; import { setExtentMemoryLimit } from "../common/ConfigurationBase"; import BlobEnvironment from "./BlobEnvironment"; import { AzuriteTelemetryClient } from "../common/Telemetry"; +import DfsConfiguration from "./DfsConfiguration"; +import DfsServer from "./DfsServer"; // tslint:disable:no-console -function shutdown(server: BlobServer | SqlBlobServer) { +function shutdown(server: BlobServer | SqlBlobServer, dfsServer: DfsServer) { const beforeCloseMessage = `Azurite Blob service is closing...`; const afterCloseMessage = `Azurite Blob service successfully closed`; + const dfsBeforeCloseMessage = `Azurite DFS service is closing...`; + const dfsAfterCloseMessage = `Azurite DFS service successfully closed`; AzuriteTelemetryClient.TraceStopEvent("Blob"); console.log(beforeCloseMessage); server.close().then(() => { console.log(afterCloseMessage); }); + + console.log(dfsBeforeCloseMessage); + dfsServer.close().then(() => { + console.log(dfsAfterCloseMessage); + }); } /** @@ -27,6 +36,24 @@ async function main() { const blobServerFactory = new BlobServerFactory(); const server = await blobServerFactory.createServer(); const config = server.config; + const env = new BlobEnvironment(); + const dfsConfig = new DfsConfiguration( + env.dfsHost(), + env.dfsPort(), + env.blobKeepAliveTimeout(), + env.cert(), + env.key(), + env.pwd() + ); + const enableHns = env.enableHierarchicalNamespace(); + const dfsServer = new DfsServer( + dfsConfig, + (server as BlobServer).metadataStore, + (server as BlobServer).extentStore, + (server as BlobServer).accountDataStore, + undefined, + enableHns + ); // We use logger singleton as global debugger logger to track detailed outputs cross layers // Note that, debug log is different from access log which is only available in request handler layer to @@ -34,7 +61,6 @@ async function main() { // Enable debug log by default before first release for debugging purpose Logger.configLogger(config.enableDebugLog, config.debugLogFilePath); - let env = new BlobEnvironment(); setExtentMemoryLimit(env, true); // Start server @@ -45,20 +71,28 @@ async function main() { console.log( `Azurite Blob service successfully listens on ${server.getHttpServerAddress()}` ); - + + console.log( + `Azurite DFS service is starting on ${dfsConfig.host}:${dfsConfig.port}` + ); + await dfsServer.start(); + console.log( + `Azurite DFS service successfully listens on ${dfsServer.getHttpServerAddress()}` + ); + const location = await env.location(); - AzuriteTelemetryClient.init(location, !env.disableTelemetry(), env); + AzuriteTelemetryClient.init(location, !env.disableTelemetry(), env); await AzuriteTelemetryClient.TraceStartEvent("Blob"); // Handle close event process .once("message", (msg) => { if (msg === "shutdown") { - shutdown(server); + shutdown(server, dfsServer); } }) - .once("SIGINT", () => shutdown(server)) - .once("SIGTERM", () => shutdown(server)); + .once("SIGINT", () => shutdown(server, dfsServer)) + .once("SIGTERM", () => shutdown(server, dfsServer)); } main().catch((err) => { diff --git a/src/blob/persistence/IBlobMetadataStore.ts b/src/blob/persistence/IBlobMetadataStore.ts index fb933f8df..780dc6ffc 100644 --- a/src/blob/persistence/IBlobMetadataStore.ts +++ b/src/blob/persistence/IBlobMetadataStore.ts @@ -1160,6 +1160,120 @@ export interface IBlobMetadataStore options: Models.AppendBlobSealOptionalParams, ): Promise; + /** + * Atomically rename a single blob (metadata-only, no extent copy). + * + * @param {Context} context + * @param {string} account + * @param {string} sourceContainer + * @param {string} sourceBlob + * @param {string} destContainer + * @param {string} destBlob + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + renameBlob( + context: Context, + account: string, + sourceContainer: string, + sourceBlob: string, + destContainer: string, + destBlob: string + ): Promise; + + /** + * Atomically rename all blobs matching a prefix (for directory rename). + * + * @param {Context} context + * @param {string} account + * @param {string} sourceContainer + * @param {string} sourcePrefix + * @param {string} destContainer + * @param {string} destPrefix + * @returns {Promise} + * @memberof IBlobMetadataStore + */ + renameBlobsByPrefix( + context: Context, + account: string, + sourceContainer: string, + sourcePrefix: string, + destContainer: string, + destPrefix: string + ): Promise; + + // --------------------------------------------------------------------------- + // HNS (Hierarchical Namespace) parent-child hierarchy methods + // --------------------------------------------------------------------------- + + /** + * Register a path in the HNS hierarchy table. + * Called when creating a file or directory via DFS. + */ + registerHnsPath( + context: Context, + account: string, + container: string, + path: string, + parentPath: string | null, + isDirectory: boolean + ): Promise; + + /** + * Unregister a path from the HNS hierarchy table. + * Called when deleting a file or directory via DFS. + */ + unregisterHnsPath( + context: Context, + account: string, + container: string, + path: string + ): Promise; + + /** + * Unregister all paths under a prefix from the HNS hierarchy table. + * Called when recursively deleting a directory via DFS. + */ + unregisterHnsPathsByPrefix( + context: Context, + account: string, + container: string, + prefix: string + ): Promise; + + /** + * Rename paths in the HNS hierarchy table. + * Called when renaming a file or directory via DFS. + */ + renameHnsPaths( + context: Context, + account: string, + sourceContainer: string, + sourcePath: string, + destContainer: string, + destPath: string + ): Promise; + + /** + * Check if a directory has any direct children in the HNS hierarchy. + * Returns true if the directory is empty (no children). + */ + isHnsDirectoryEmpty( + context: Context, + account: string, + container: string, + directoryPath: string + ): Promise; + + /** + * Check if a path exists in the HNS hierarchy table. + */ + hnsPathExists( + context: Context, + account: string, + container: string, + path: string + ): Promise; } export default IBlobMetadataStore; diff --git a/src/blob/persistence/LokiBlobMetadataStore.ts b/src/blob/persistence/LokiBlobMetadataStore.ts index 65a953ebe..4cece9fec 100644 --- a/src/blob/persistence/LokiBlobMetadataStore.ts +++ b/src/blob/persistence/LokiBlobMetadataStore.ts @@ -105,6 +105,7 @@ export default class LokiBlobMetadataStore private readonly CONTAINERS_COLLECTION = "$CONTAINERS_COLLECTION$"; private readonly BLOBS_COLLECTION = "$BLOBS_COLLECTION$"; private readonly BLOCKS_COLLECTION = "$BLOCKS_COLLECTION$"; + private readonly HNS_HIERARCHY_COLLECTION = "$HNS_HIERARCHY$"; private readonly pageBlobRangesManager = new PageBlobRangesManager(); @@ -177,6 +178,13 @@ export default class LokiBlobMetadataStore }); } + // Create HNS hierarchy collection if not exists (parent-child relationships) + if (this.db.getCollection(this.HNS_HIERARCHY_COLLECTION) === null) { + this.db.addCollection(this.HNS_HIERARCHY_COLLECTION, { + indices: ["accountName", "containerName", "path", "parentPath"] + }); + } + await new Promise((resolve, reject) => { this.db.saveDatabase((err) => { if (err) { @@ -3562,4 +3570,197 @@ export default class LokiBlobMetadataStore return doc.properties; } + + public async renameBlob( + context: Context, + account: string, + sourceContainer: string, + sourceBlob: string, + destContainer: string, + destBlob: string + ): Promise { + const coll = this.db.getCollection(this.BLOBS_COLLECTION); + const doc = coll.findOne({ + accountName: account, + containerName: sourceContainer, + name: sourceBlob, + snapshot: "" + }); + + if (!doc) { + throw StorageErrorFactory.getBlobNotFound(context.contextId); + } + + doc.containerName = destContainer; + doc.name = destBlob; + doc.properties.lastModified = context.startTime!; + doc.properties.etag = newEtag(); + coll.update(doc); + + return doc.properties; + } + + public async renameBlobsByPrefix( + context: Context, + account: string, + sourceContainer: string, + sourcePrefix: string, + destContainer: string, + destPrefix: string + ): Promise { + const coll = this.db.getCollection(this.BLOBS_COLLECTION); + const docs = coll.find({ + accountName: account, + containerName: sourceContainer, + name: { $regex: new RegExp(`^${this.escapeRegExp(sourcePrefix)}`) } + }); + + for (const doc of docs) { + doc.containerName = destContainer; + doc.name = destPrefix + doc.name.substring(sourcePrefix.length); + doc.properties.lastModified = context.startTime!; + doc.properties.etag = newEtag(); + coll.update(doc); + } + } + + private escapeRegExp(str: string): string { + return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + } + + // --------------------------------------------------------------------------- + // HNS hierarchy methods + // --------------------------------------------------------------------------- + + public async registerHnsPath( + _context: Context, + account: string, + container: string, + path: string, + parentPath: string | null, + isDirectory: boolean + ): Promise { + const coll = this.db.getCollection(this.HNS_HIERARCHY_COLLECTION); + const existing = coll.findOne({ + accountName: account, + containerName: container, + path + }); + if (existing) { + existing.parentPath = parentPath; + existing.isDirectory = isDirectory; + coll.update(existing); + } else { + coll.insert({ + accountName: account, + containerName: container, + path, + parentPath, + isDirectory + }); + } + } + + public async unregisterHnsPath( + _context: Context, + account: string, + container: string, + path: string + ): Promise { + const coll = this.db.getCollection(this.HNS_HIERARCHY_COLLECTION); + coll.findAndRemove({ + accountName: account, + containerName: container, + path + }); + } + + public async unregisterHnsPathsByPrefix( + _context: Context, + account: string, + container: string, + prefix: string + ): Promise { + const coll = this.db.getCollection(this.HNS_HIERARCHY_COLLECTION); + coll.findAndRemove({ + accountName: account, + containerName: container, + path: { $regex: new RegExp(`^${this.escapeRegExp(prefix)}`) } + }); + } + + public async renameHnsPaths( + _context: Context, + account: string, + sourceContainer: string, + sourcePath: string, + destContainer: string, + destPath: string + ): Promise { + const coll = this.db.getCollection(this.HNS_HIERARCHY_COLLECTION); + + // Rename the path itself + const doc = coll.findOne({ + accountName: account, + containerName: sourceContainer, + path: sourcePath + }); + if (doc) { + doc.containerName = destContainer; + doc.path = destPath; + doc.parentPath = destPath.includes("/") + ? destPath.substring(0, destPath.lastIndexOf("/")) + : null; + coll.update(doc); + } + + // Rename all children (paths starting with sourcePath/) + const sourcePrefix = sourcePath + "/"; + const destPrefix = destPath + "/"; + const children = coll.find({ + accountName: account, + containerName: sourceContainer, + path: { $regex: new RegExp(`^${this.escapeRegExp(sourcePrefix)}`) } + }); + for (const child of children) { + const relativePath = child.path.substring(sourcePrefix.length); + child.containerName = destContainer; + child.path = destPrefix + relativePath; + // Update parentPath: replace source prefix with dest prefix + if (child.parentPath && child.parentPath.startsWith(sourcePath)) { + child.parentPath = destPath + child.parentPath.substring(sourcePath.length); + } + coll.update(child); + } + } + + public async isHnsDirectoryEmpty( + _context: Context, + account: string, + container: string, + directoryPath: string + ): Promise { + const coll = this.db.getCollection(this.HNS_HIERARCHY_COLLECTION); + const count = coll.count({ + accountName: account, + containerName: container, + parentPath: directoryPath + }); + return count === 0; + } + + public async hnsPathExists( + _context: Context, + account: string, + container: string, + path: string + ): Promise { + const coll = this.db.getCollection(this.HNS_HIERARCHY_COLLECTION); + const doc = coll.findOne({ + accountName: account, + containerName: container, + path + }); + return doc !== null; + } } diff --git a/src/blob/persistence/SqlBlobMetadataStore.ts b/src/blob/persistence/SqlBlobMetadataStore.ts index 73c2cf9c3..368b65c4e 100644 --- a/src/blob/persistence/SqlBlobMetadataStore.ts +++ b/src/blob/persistence/SqlBlobMetadataStore.ts @@ -79,6 +79,7 @@ class ServicesModel extends Model { } class ContainersModel extends Model { } class BlobsModel extends Model { } class BlocksModel extends Model { } +class HnsHierarchyModel extends Model { } // class PagesModel extends Model {} interface IBlobContentProperties { @@ -370,6 +371,53 @@ export default class SqlBlobMetadataStore implements IBlobMetadataStore { } ); + // HNS hierarchy table: parent-child relationships for hierarchical namespace + HnsHierarchyModel.init( + { + id: { + type: INTEGER.UNSIGNED, + primaryKey: true, + autoIncrement: true + }, + accountName: { + type: "VARCHAR(64)", + allowNull: false + }, + containerName: { + type: "VARCHAR(255)", + allowNull: false + }, + path: { + type: "VARCHAR(1024)", + allowNull: false + }, + parentPath: { + type: "VARCHAR(1024)", + allowNull: true + }, + isDirectory: { + type: BOOLEAN, + allowNull: false, + defaultValue: false + } + }, + { + sequelize: this.sequelize, + modelName: "HnsHierarchy", + tableName: "HnsHierarchy", + timestamps: false, + indexes: [ + { + unique: true, + fields: ["accountName", "containerName", "path"] + }, + { + fields: ["accountName", "containerName", "parentPath"] + } + ] + } + ); + // TODO: sync() is only for development purpose, use migration for production await this.sequelize.sync(); @@ -3576,4 +3624,223 @@ export default class SqlBlobMetadataStore implements IBlobMetadataStore { ): Promise { throw new NotImplementedinSQLError(context.contextId); } + + public async renameBlob( + context: Context, + account: string, + sourceContainer: string, + sourceBlob: string, + destContainer: string, + destBlob: string + ): Promise { + return this.sequelize.transaction(async (t) => { + const now = new Date(); + const etag = newEtag(); + const [affectedCount] = await BlobsModel.update( + { + containerName: destContainer, + blobName: destBlob, + lastModified: now, + etag + }, + { + where: { + accountName: account, + containerName: sourceContainer, + blobName: sourceBlob, + snapshot: "" + }, + transaction: t + } + ); + + if (affectedCount === 0) { + throw StorageErrorFactory.getBlobNotFound(context.contextId); + } + + return { + lastModified: now, + etag + } as Models.BlobPropertiesInternal; + }); + } + + public async renameBlobsByPrefix( + context: Context, + account: string, + sourceContainer: string, + sourcePrefix: string, + destContainer: string, + destPrefix: string + ): Promise { + await this.sequelize.transaction(async (t) => { + const now = new Date(); + const etag = newEtag(); + // Use Sequelize literal for SQL REPLACE to atomically rename all matching blobs + await BlobsModel.update( + { + containerName: destContainer, + blobName: this.sequelize.literal( + `REPLACE("blobName", ${this.sequelize.escape(sourcePrefix)}, ${this.sequelize.escape(destPrefix)})` + ), + lastModified: now, + etag + } as any, + { + where: { + accountName: account, + containerName: sourceContainer, + blobName: { [Op.like]: `${sourcePrefix}%` } + }, + transaction: t + } + ); + }); + } + + // --------------------------------------------------------------------------- + // HNS hierarchy methods + // --------------------------------------------------------------------------- + + public async registerHnsPath( + _context: Context, + account: string, + container: string, + path: string, + parentPath: string | null, + isDirectory: boolean + ): Promise { + await HnsHierarchyModel.upsert({ + accountName: account, + containerName: container, + path, + parentPath, + isDirectory + }); + } + + public async unregisterHnsPath( + _context: Context, + account: string, + container: string, + path: string + ): Promise { + await HnsHierarchyModel.destroy({ + where: { + accountName: account, + containerName: container, + path + } + }); + } + + public async unregisterHnsPathsByPrefix( + _context: Context, + account: string, + container: string, + prefix: string + ): Promise { + await HnsHierarchyModel.destroy({ + where: { + accountName: account, + containerName: container, + path: { [Op.like]: `${prefix}%` } + } + }); + } + + public async renameHnsPaths( + _context: Context, + account: string, + sourceContainer: string, + sourcePath: string, + destContainer: string, + destPath: string + ): Promise { + await this.sequelize.transaction(async (t) => { + // Rename the path itself + await HnsHierarchyModel.update( + { + containerName: destContainer, + path: destPath, + parentPath: destPath.includes("/") + ? destPath.substring(0, destPath.lastIndexOf("/")) + : null + }, + { + where: { + accountName: account, + containerName: sourceContainer, + path: sourcePath + }, + transaction: t + } + ); + + // Rename all children + const sourcePrefix = sourcePath + "/"; + const destPrefix = destPath + "/"; + const children = await HnsHierarchyModel.findAll({ + where: { + accountName: account, + containerName: sourceContainer, + path: { [Op.like]: `${sourcePrefix}%` } + }, + transaction: t + }); + + for (const child of children) { + const childData = child.get() as any; + const relativePath = childData.path.substring(sourcePrefix.length); + const newPath = destPrefix + relativePath; + let newParent = childData.parentPath; + if (newParent && newParent.startsWith(sourcePath)) { + newParent = destPath + newParent.substring(sourcePath.length); + } + await HnsHierarchyModel.update( + { + containerName: destContainer, + path: newPath, + parentPath: newParent + }, + { + where: { id: childData.id }, + transaction: t + } + ); + } + }); + } + + public async isHnsDirectoryEmpty( + _context: Context, + account: string, + container: string, + directoryPath: string + ): Promise { + const count = await HnsHierarchyModel.count({ + where: { + accountName: account, + containerName: container, + parentPath: directoryPath + } + }); + return count === 0; + } + + public async hnsPathExists( + _context: Context, + account: string, + container: string, + path: string + ): Promise { + const count = await HnsHierarchyModel.count({ + where: { + accountName: account, + containerName: container, + path + } + }); + return count > 0; + } } diff --git a/src/blob/utils/constants.ts b/src/blob/utils/constants.ts index c641b50de..1e97a5334 100644 --- a/src/blob/utils/constants.ts +++ b/src/blob/utils/constants.ts @@ -7,6 +7,8 @@ export const DEFAULT_BLOB_SERVER_HOST_NAME = "127.0.0.1"; // Change to 0.0.0.0 w export const DEFAULT_LIST_BLOBS_MAX_RESULTS = 5000; export const DEFAULT_LIST_CONTAINERS_MAX_RESULTS = 5000; export const DEFAULT_BLOB_LISTENING_PORT = 10000; +export const DEFAULT_DFS_LISTENING_PORT = 10004; +export const DEFAULT_DFS_SERVER_HOST_NAME = DEFAULT_BLOB_SERVER_HOST_NAME; export const IS_PRODUCTION = process.env.NODE_ENV === "production"; export const DEFAULT_BLOB_LOKI_DB_PATH = "__azurite_db_blob__.json"; export const DEFAULT_BLOB_EXTENT_LOKI_DB_PATH = @@ -30,7 +32,10 @@ export const EMULATOR_ACCOUNT_KEY = Buffer.from( export const EMULATOR_ACCOUNT_SKUNAME = Models.SkuName.StandardRAGRS; export const EMULATOR_ACCOUNT_KIND = Models.AccountKind.StorageV2; -export const EMULATOR_ACCOUNT_ISHIERARCHICALNAMESPACEENABLED = false; +export const EMULATOR_ACCOUNT_ISHIERARCHICALNAMESPACEENABLED_DEFAULT = true; +// Alias for backward compatibility — existing code imports this name +export const EMULATOR_ACCOUNT_ISHIERARCHICALNAMESPACEENABLED = + EMULATOR_ACCOUNT_ISHIERARCHICALNAMESPACEENABLED_DEFAULT; export const DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT = 5; diff --git a/src/common/ConfigurationBase.ts b/src/common/ConfigurationBase.ts index be3f0928b..036437797 100644 --- a/src/common/ConfigurationBase.ts +++ b/src/common/ConfigurationBase.ts @@ -92,9 +92,13 @@ export default abstract class ConfigurationBase { public getOAuthLevel(): undefined | OAuthLevel { if (this.oauth) { - if (this.oauth.toLowerCase() === "basic") { + const level = this.oauth.toLowerCase(); + if (level === "basic") { return OAuthLevel.BASIC; } + if (level === "acl") { + return OAuthLevel.ACL; + } } return; diff --git a/src/common/Environment.ts b/src/common/Environment.ts index e42c4fddb..b517deaee 100644 --- a/src/common/Environment.ts +++ b/src/common/Environment.ts @@ -3,7 +3,9 @@ import args from "args"; import { DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT, DEFAULT_BLOB_LISTENING_PORT, - DEFAULT_BLOB_SERVER_HOST_NAME + DEFAULT_BLOB_SERVER_HOST_NAME, + DEFAULT_DFS_LISTENING_PORT, + DEFAULT_DFS_SERVER_HOST_NAME } from "../blob/utils/constants"; import { @@ -31,6 +33,16 @@ args "Optional. Customize listening port for blob", DEFAULT_BLOB_LISTENING_PORT ) + .option( + ["", "dfsHost"], + "Optional. Customize listening address for DFS", + DEFAULT_DFS_SERVER_HOST_NAME + ) + .option( + ["", "dfsPort"], + "Optional. Customize listening port for DFS", + DEFAULT_DFS_LISTENING_PORT + ) .option( ["", "blobKeepAliveTimeout"], "Optional. Customize http keep alive timeout for blob", @@ -110,6 +122,10 @@ args .option( ["", "disableTelemetry"], "Optional. Disable telemtry collection of Azurite. If not specify this parameter Azurite will collect telemetry data by default." + ) + .option( + ["", "enableHierarchicalNamespace"], + "Optional. Enable hierarchical namespace (HNS) mode for ADLS Gen2. Default is true." ); (args as any).config.name = "azurite"; @@ -125,6 +141,14 @@ export default class Environment implements IEnvironment { return this.flags.blobPort; } + public dfsHost(): string | undefined { + return this.flags.dfsHost; + } + + public dfsPort(): number | undefined { + return this.flags.dfsPort; + } + public blobKeepAliveTimeout(): number | undefined { return this.flags.blobKeepAliveTimeout; } @@ -222,6 +246,13 @@ export default class Environment implements IEnvironment { return this.flags.extentMemoryLimit; } + public enableHierarchicalNamespace(): boolean { + if (this.flags.enableHierarchicalNamespace !== undefined) { + return this.flags.enableHierarchicalNamespace !== false; + } + return true; // default enabled + } + public disableTelemetry(): boolean { if (this.flags.disableTelemetry !== undefined) { return true; diff --git a/src/common/Telemetry.ts b/src/common/Telemetry.ts index 20379e3e2..5e568c3e3 100644 --- a/src/common/Telemetry.ts +++ b/src/common/Telemetry.ts @@ -11,7 +11,7 @@ import * as fs from "fs"; import uuid from "uuid"; import { join } from "path"; import logger from "./Logger"; -import { DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT, DEFAULT_BLOB_LISTENING_PORT, DEFAULT_BLOB_SERVER_HOST_NAME } from "../blob/utils/constants"; +import { DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT, DEFAULT_BLOB_LISTENING_PORT, DEFAULT_BLOB_SERVER_HOST_NAME, DEFAULT_DFS_LISTENING_PORT } from "../blob/utils/constants"; import { DEFAULT_QUEUE_LISTENING_PORT } from "../queue/utils/constants"; import { DEFAULT_TABLE_LISTENING_PORT } from "../table/utils/constants"; @@ -355,7 +355,7 @@ export class AzuriteTelemetryClient { { parameters += "AZURITE_DB,"; } - let longParameters = ["blobHost","queueHost","tableHost","blobPort","queuePort","tablePort","blobKeepAliveTimeout","queueKeepAliveTimeout","tableKeepAliveTimeout","location","cert","key","pwd","oauth","extentMemoryLimit","debug","silent","loose","skipApiVersionCheck","disableProductStyleUrl","inMemoryPersistence","disableTelemetry"]; + let longParameters = ["blobHost","dfsHost","queueHost","tableHost","blobPort","dfsPort","queuePort","tablePort","blobKeepAliveTimeout","queueKeepAliveTimeout","tableKeepAliveTimeout","location","cert","key","pwd","oauth","extentMemoryLimit","debug","silent","loose","skipApiVersionCheck","disableProductStyleUrl","inMemoryPersistence","disableTelemetry"]; let shortParameters: { [string: string]: any } = {"d": "debug", "l": "location", "L": "loose", "s": "silent"}; if (AzuriteTelemetryClient.isVSC) // VSC @@ -377,6 +377,7 @@ export class AzuriteTelemetryClient { && !(flag.endsWith("Host") && value === DEFAULT_BLOB_SERVER_HOST_NAME) && !(flag.endsWith("KeepAliveTimeout") && value === DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT) && !(flag == "blobPort" && value === DEFAULT_BLOB_LISTENING_PORT) + && !(flag == "dfsPort" && value === DEFAULT_DFS_LISTENING_PORT) && !(flag == "queuePort" && value === DEFAULT_QUEUE_LISTENING_PORT) && !(flag == "tablePort" && value === DEFAULT_TABLE_LISTENING_PORT)) { diff --git a/src/common/VSCEnvironment.ts b/src/common/VSCEnvironment.ts index 0bcff08f5..c4b9d63c3 100644 --- a/src/common/VSCEnvironment.ts +++ b/src/common/VSCEnvironment.ts @@ -15,6 +15,14 @@ export default class VSCEnvironment implements IEnvironment { return this.workspaceConfiguration.get("blobPort"); } + public dfsHost(): string | undefined { + return this.workspaceConfiguration.get("dfsHost"); + } + + public dfsPort(): number | undefined { + return this.workspaceConfiguration.get("dfsPort"); + } + public blobKeepAliveTimeout(): number | undefined { return this.workspaceConfiguration.get("blobKeepAliveTimeout"); } @@ -135,4 +143,10 @@ export default class VSCEnvironment implements IEnvironment { this.workspaceConfiguration.get("disableTelemetry") || false ); } + + public enableHierarchicalNamespace(): boolean { + return ( + this.workspaceConfiguration.get("enableHierarchicalNamespace") ?? true + ); + } } diff --git a/src/common/models.ts b/src/common/models.ts index e48961e35..2b3c15e8d 100644 --- a/src/common/models.ts +++ b/src/common/models.ts @@ -1,3 +1,4 @@ export enum OAuthLevel { - BASIC // Phase 1 + BASIC, // Phase 1: Token format/lifetime/issuer validation only + ACL // Phase 3: Token validation + ACL enforcement on DFS paths } diff --git a/swagger/dfs-storage-2023-11-03.json b/swagger/dfs-storage-2023-11-03.json new file mode 100644 index 000000000..a80377145 --- /dev/null +++ b/swagger/dfs-storage-2023-11-03.json @@ -0,0 +1,768 @@ +{ + "swagger": "2.0", + "info": { + "title": "Azure Data Lake Storage REST API", + "version": "2023-11-03", + "description": "Azure Data Lake Storage provides an ADLS Gen2 (DFS) REST API for hierarchical namespace operations." + }, + "x-ms-parameterized-host": { + "hostTemplate": "{url}", + "useSchemePrefix": false, + "positionInOperation": "first", + "parameters": [ + { + "$ref": "#/parameters/Url" + } + ] + }, + "schemes": ["https"], + "consumes": ["application/json"], + "produces": ["application/json"], + "paths": {}, + "x-ms-paths": { + "/?resource=account": { + "get": { + "operationId": "Filesystem_List", + "summary": "List Filesystems", + "description": "List filesystems and their properties in given account.", + "parameters": [ + { "$ref": "#/parameters/Prefix" }, + { "$ref": "#/parameters/Continuation" }, + { "$ref": "#/parameters/MaxResults" }, + { "$ref": "#/parameters/ApiVersionParameter" }, + { "$ref": "#/parameters/ClientRequestId" } + ], + "responses": { + "200": { + "description": "OK", + "headers": { + "x-ms-continuation": { "type": "string" }, + "x-ms-request-id": { "type": "string" }, + "x-ms-version": { "type": "string" }, + "Date": { "type": "string", "format": "date-time-rfc1123" } + }, + "schema": { "$ref": "#/definitions/FilesystemList" } + }, + "default": { + "description": "Failure", + "schema": { "$ref": "#/definitions/StorageError" } + } + } + } + }, + "/{filesystem}?resource=filesystem": { + "put": { + "operationId": "Filesystem_Create", + "summary": "Create Filesystem", + "parameters": [ + { "$ref": "#/parameters/Filesystem" }, + { "$ref": "#/parameters/Properties" }, + { "$ref": "#/parameters/ApiVersionParameter" }, + { "$ref": "#/parameters/ClientRequestId" } + ], + "responses": { + "201": { + "description": "Created", + "headers": { + "ETag": { "type": "string" }, + "Last-Modified": { "type": "string", "format": "date-time-rfc1123" }, + "x-ms-namespace-enabled": { "type": "string" }, + "x-ms-request-id": { "type": "string" }, + "x-ms-version": { "type": "string" } + } + }, + "default": { + "description": "Failure", + "schema": { "$ref": "#/definitions/StorageError" } + } + } + }, + "delete": { + "operationId": "Filesystem_Delete", + "summary": "Delete Filesystem", + "parameters": [ + { "$ref": "#/parameters/Filesystem" }, + { "$ref": "#/parameters/IfModifiedSince" }, + { "$ref": "#/parameters/IfUnmodifiedSince" }, + { "$ref": "#/parameters/ApiVersionParameter" }, + { "$ref": "#/parameters/ClientRequestId" } + ], + "responses": { + "202": { + "description": "Accepted", + "headers": { + "x-ms-request-id": { "type": "string" }, + "x-ms-version": { "type": "string" } + } + }, + "default": { + "description": "Failure", + "schema": { "$ref": "#/definitions/StorageError" } + } + } + }, + "head": { + "operationId": "Filesystem_GetProperties", + "summary": "Get Filesystem Properties", + "parameters": [ + { "$ref": "#/parameters/Filesystem" }, + { "$ref": "#/parameters/ApiVersionParameter" }, + { "$ref": "#/parameters/ClientRequestId" } + ], + "responses": { + "200": { + "description": "OK", + "headers": { + "ETag": { "type": "string" }, + "Last-Modified": { "type": "string", "format": "date-time-rfc1123" }, + "x-ms-properties": { "type": "string" }, + "x-ms-namespace-enabled": { "type": "string" }, + "x-ms-request-id": { "type": "string" }, + "x-ms-version": { "type": "string" } + } + }, + "default": { + "description": "Failure", + "schema": { "$ref": "#/definitions/StorageError" } + } + } + }, + "patch": { + "operationId": "Filesystem_SetProperties", + "summary": "Set Filesystem Properties", + "parameters": [ + { "$ref": "#/parameters/Filesystem" }, + { "$ref": "#/parameters/Properties" }, + { "$ref": "#/parameters/IfModifiedSince" }, + { "$ref": "#/parameters/IfUnmodifiedSince" }, + { "$ref": "#/parameters/ApiVersionParameter" }, + { "$ref": "#/parameters/ClientRequestId" } + ], + "responses": { + "200": { + "description": "OK", + "headers": { + "ETag": { "type": "string" }, + "Last-Modified": { "type": "string", "format": "date-time-rfc1123" }, + "x-ms-request-id": { "type": "string" }, + "x-ms-version": { "type": "string" } + } + }, + "default": { + "description": "Failure", + "schema": { "$ref": "#/definitions/StorageError" } + } + } + }, + "get": { + "operationId": "Filesystem_ListPaths", + "summary": "List Paths", + "parameters": [ + { "$ref": "#/parameters/Filesystem" }, + { "$ref": "#/parameters/Directory" }, + { "$ref": "#/parameters/RecursiveRequired" }, + { "$ref": "#/parameters/Continuation" }, + { "$ref": "#/parameters/MaxResults" }, + { "$ref": "#/parameters/Upn" }, + { "$ref": "#/parameters/ApiVersionParameter" }, + { "$ref": "#/parameters/ClientRequestId" } + ], + "responses": { + "200": { + "description": "OK", + "headers": { + "x-ms-continuation": { "type": "string" }, + "x-ms-request-id": { "type": "string" }, + "x-ms-version": { "type": "string" } + }, + "schema": { "$ref": "#/definitions/PathList" } + }, + "default": { + "description": "Failure", + "schema": { "$ref": "#/definitions/StorageError" } + } + } + } + }, + "/{filesystem}/{path}": { + "put": { + "operationId": "Path_Create", + "summary": "Create or Rename Path", + "parameters": [ + { "$ref": "#/parameters/Filesystem" }, + { "$ref": "#/parameters/Path" }, + { "$ref": "#/parameters/Resource" }, + { "$ref": "#/parameters/Continuation" }, + { "$ref": "#/parameters/RenameSource" }, + { "$ref": "#/parameters/Properties" }, + { "$ref": "#/parameters/Permissions" }, + { "$ref": "#/parameters/Umask" }, + { "$ref": "#/parameters/IfMatch" }, + { "$ref": "#/parameters/IfNoneMatch" }, + { "$ref": "#/parameters/IfModifiedSince" }, + { "$ref": "#/parameters/IfUnmodifiedSince" }, + { "$ref": "#/parameters/SourceIfMatch" }, + { "$ref": "#/parameters/SourceIfNoneMatch" }, + { "$ref": "#/parameters/SourceIfModifiedSince" }, + { "$ref": "#/parameters/SourceIfUnmodifiedSince" }, + { "$ref": "#/parameters/ApiVersionParameter" }, + { "$ref": "#/parameters/ClientRequestId" } + ], + "responses": { + "201": { + "description": "Created", + "headers": { + "ETag": { "type": "string" }, + "Last-Modified": { "type": "string", "format": "date-time-rfc1123" }, + "x-ms-continuation": { "type": "string" }, + "Content-Length": { "type": "integer", "format": "int64" }, + "x-ms-request-id": { "type": "string" }, + "x-ms-version": { "type": "string" } + } + }, + "default": { + "description": "Failure", + "schema": { "$ref": "#/definitions/StorageError" } + } + } + }, + "head": { + "operationId": "Path_GetProperties", + "summary": "Get Properties / Get Access Control", + "parameters": [ + { "$ref": "#/parameters/Filesystem" }, + { "$ref": "#/parameters/Path" }, + { "$ref": "#/parameters/Action_GetAccessControl" }, + { "$ref": "#/parameters/Upn" }, + { "$ref": "#/parameters/LeaseIdOptional" }, + { "$ref": "#/parameters/IfMatch" }, + { "$ref": "#/parameters/IfNoneMatch" }, + { "$ref": "#/parameters/IfModifiedSince" }, + { "$ref": "#/parameters/IfUnmodifiedSince" }, + { "$ref": "#/parameters/ApiVersionParameter" }, + { "$ref": "#/parameters/ClientRequestId" } + ], + "responses": { + "200": { + "description": "OK", + "headers": { + "ETag": { "type": "string" }, + "Last-Modified": { "type": "string", "format": "date-time-rfc1123" }, + "x-ms-resource-type": { "type": "string" }, + "x-ms-properties": { "type": "string" }, + "x-ms-owner": { "type": "string" }, + "x-ms-group": { "type": "string" }, + "x-ms-permissions": { "type": "string" }, + "x-ms-acl": { "type": "string" }, + "x-ms-lease-duration": { "type": "string" }, + "x-ms-lease-state": { "type": "string" }, + "x-ms-lease-status": { "type": "string" }, + "Content-Length": { "type": "integer", "format": "int64" }, + "Content-Type": { "type": "string" }, + "x-ms-request-id": { "type": "string" }, + "x-ms-version": { "type": "string" } + } + }, + "default": { + "description": "Failure", + "schema": { "$ref": "#/definitions/StorageError" } + } + } + }, + "get": { + "operationId": "Path_Read", + "summary": "Read File", + "parameters": [ + { "$ref": "#/parameters/Filesystem" }, + { "$ref": "#/parameters/Path" }, + { "$ref": "#/parameters/Range" }, + { "$ref": "#/parameters/LeaseIdOptional" }, + { "$ref": "#/parameters/IfMatch" }, + { "$ref": "#/parameters/IfNoneMatch" }, + { "$ref": "#/parameters/IfModifiedSince" }, + { "$ref": "#/parameters/IfUnmodifiedSince" }, + { "$ref": "#/parameters/ApiVersionParameter" }, + { "$ref": "#/parameters/ClientRequestId" } + ], + "responses": { + "200": { + "description": "OK", + "headers": { + "Accept-Ranges": { "type": "string" }, + "Content-Range": { "type": "string" }, + "ETag": { "type": "string" }, + "Last-Modified": { "type": "string", "format": "date-time-rfc1123" }, + "x-ms-resource-type": { "type": "string" }, + "x-ms-properties": { "type": "string" }, + "x-ms-lease-duration": { "type": "string" }, + "x-ms-lease-state": { "type": "string" }, + "x-ms-lease-status": { "type": "string" }, + "Content-Length": { "type": "integer", "format": "int64" }, + "Content-Type": { "type": "string" }, + "x-ms-request-id": { "type": "string" }, + "x-ms-version": { "type": "string" } + }, + "schema": { "type": "object", "format": "file" } + }, + "default": { + "description": "Failure", + "schema": { "$ref": "#/definitions/StorageError" } + } + } + }, + "patch": { + "operationId": "Path_Update", + "summary": "Append/Flush/SetAccessControl", + "parameters": [ + { "$ref": "#/parameters/Filesystem" }, + { "$ref": "#/parameters/Path" }, + { "$ref": "#/parameters/Action_Required" }, + { "$ref": "#/parameters/Mode" }, + { "$ref": "#/parameters/Position" }, + { "$ref": "#/parameters/RetainUncommittedData" }, + { "$ref": "#/parameters/Close" }, + { "$ref": "#/parameters/ContentLength" }, + { "$ref": "#/parameters/ContentMD5" }, + { "$ref": "#/parameters/Properties" }, + { "$ref": "#/parameters/Owner" }, + { "$ref": "#/parameters/Group" }, + { "$ref": "#/parameters/PermissionsOptional" }, + { "$ref": "#/parameters/Acl" }, + { "$ref": "#/parameters/LeaseIdOptional" }, + { "$ref": "#/parameters/IfMatch" }, + { "$ref": "#/parameters/IfNoneMatch" }, + { "$ref": "#/parameters/IfModifiedSince" }, + { "$ref": "#/parameters/IfUnmodifiedSince" }, + { "$ref": "#/parameters/MaxRecords" }, + { "$ref": "#/parameters/Continuation" }, + { "$ref": "#/parameters/ApiVersionParameter" }, + { "$ref": "#/parameters/ClientRequestId" } + ], + "responses": { + "200": { + "description": "SetAccessControl/Flush success", + "headers": { + "ETag": { "type": "string" }, + "Last-Modified": { "type": "string", "format": "date-time-rfc1123" }, + "x-ms-request-id": { "type": "string" }, + "x-ms-version": { "type": "string" } + } + }, + "202": { + "description": "Append accepted", + "headers": { + "x-ms-request-id": { "type": "string" }, + "x-ms-version": { "type": "string" } + } + }, + "default": { + "description": "Failure", + "schema": { "$ref": "#/definitions/StorageError" } + } + } + }, + "delete": { + "operationId": "Path_Delete", + "summary": "Delete Path", + "parameters": [ + { "$ref": "#/parameters/Filesystem" }, + { "$ref": "#/parameters/Path" }, + { "$ref": "#/parameters/RecursiveOptional" }, + { "$ref": "#/parameters/Continuation" }, + { "$ref": "#/parameters/LeaseIdOptional" }, + { "$ref": "#/parameters/IfMatch" }, + { "$ref": "#/parameters/IfNoneMatch" }, + { "$ref": "#/parameters/IfModifiedSince" }, + { "$ref": "#/parameters/IfUnmodifiedSince" }, + { "$ref": "#/parameters/ApiVersionParameter" }, + { "$ref": "#/parameters/ClientRequestId" } + ], + "responses": { + "200": { + "description": "OK", + "headers": { + "x-ms-continuation": { "type": "string" }, + "x-ms-request-id": { "type": "string" }, + "x-ms-version": { "type": "string" } + } + }, + "default": { + "description": "Failure", + "schema": { "$ref": "#/definitions/StorageError" } + } + } + } + }, + "/{filesystem}/{path}?comp=lease": { + "post": { + "operationId": "Path_Lease", + "summary": "Lease Path", + "parameters": [ + { "$ref": "#/parameters/Filesystem" }, + { "$ref": "#/parameters/Path" }, + { "$ref": "#/parameters/LeaseAction" }, + { "$ref": "#/parameters/LeaseDuration" }, + { "$ref": "#/parameters/LeaseBreakPeriod" }, + { "$ref": "#/parameters/LeaseIdOptional" }, + { "$ref": "#/parameters/ProposedLeaseId" }, + { "$ref": "#/parameters/IfMatch" }, + { "$ref": "#/parameters/IfNoneMatch" }, + { "$ref": "#/parameters/IfModifiedSince" }, + { "$ref": "#/parameters/IfUnmodifiedSince" }, + { "$ref": "#/parameters/ApiVersionParameter" }, + { "$ref": "#/parameters/ClientRequestId" } + ], + "responses": { + "200": { + "description": "Lease renewed/changed/released", + "headers": { + "ETag": { "type": "string" }, + "Last-Modified": { "type": "string", "format": "date-time-rfc1123" }, + "x-ms-lease-id": { "type": "string" }, + "x-ms-request-id": { "type": "string" }, + "x-ms-version": { "type": "string" } + } + }, + "201": { + "description": "Lease acquired", + "headers": { + "ETag": { "type": "string" }, + "Last-Modified": { "type": "string", "format": "date-time-rfc1123" }, + "x-ms-lease-id": { "type": "string" }, + "x-ms-request-id": { "type": "string" }, + "x-ms-version": { "type": "string" } + } + }, + "202": { + "description": "Lease broken", + "headers": { + "ETag": { "type": "string" }, + "Last-Modified": { "type": "string", "format": "date-time-rfc1123" }, + "x-ms-lease-time": { "type": "integer" }, + "x-ms-request-id": { "type": "string" }, + "x-ms-version": { "type": "string" } + } + }, + "default": { + "description": "Failure", + "schema": { "$ref": "#/definitions/StorageError" } + } + } + } + } + }, + "definitions": { + "StorageError": { + "type": "object", + "properties": { + "error": { + "type": "object", + "properties": { + "code": { "type": "string" }, + "message": { "type": "string" } + } + } + } + }, + "FilesystemList": { + "type": "object", + "properties": { + "filesystems": { + "type": "array", + "items": { "$ref": "#/definitions/Filesystem" } + } + } + }, + "Filesystem": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "lastModified": { "type": "string", "format": "date-time-rfc1123" }, + "eTag": { "type": "string" } + } + }, + "PathList": { + "type": "object", + "properties": { + "paths": { + "type": "array", + "items": { "$ref": "#/definitions/PathItem" } + } + } + }, + "PathItem": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "isDirectory": { "type": "boolean" }, + "lastModified": { "type": "string", "format": "date-time-rfc1123" }, + "eTag": { "type": "string" }, + "contentLength": { "type": "integer", "format": "int64" }, + "owner": { "type": "string" }, + "group": { "type": "string" }, + "permissions": { "type": "string" } + } + }, + "SetAccessControlRecursiveResponse": { + "type": "object", + "properties": { + "directoriesSuccessful": { "type": "integer" }, + "filesSuccessful": { "type": "integer" }, + "failureCount": { "type": "integer" }, + "failedEntries": { + "type": "array", + "items": { "$ref": "#/definitions/AclFailedEntry" } + } + } + }, + "AclFailedEntry": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "type": { "type": "string" }, + "errorMessage": { "type": "string" } + } + } + }, + "parameters": { + "Url": { + "name": "url", + "in": "path", + "required": true, + "type": "string", + "x-ms-skip-url-encoding": true + }, + "Filesystem": { + "name": "filesystem", + "in": "path", + "required": true, + "type": "string" + }, + "Path": { + "name": "path", + "in": "path", + "required": true, + "type": "string" + }, + "Resource": { + "name": "resource", + "in": "query", + "type": "string", + "enum": ["filesystem", "file", "directory"] + }, + "Directory": { + "name": "directory", + "in": "query", + "type": "string" + }, + "RecursiveRequired": { + "name": "recursive", + "in": "query", + "required": true, + "type": "boolean" + }, + "RecursiveOptional": { + "name": "recursive", + "in": "query", + "type": "boolean" + }, + "Continuation": { + "name": "continuation", + "in": "query", + "type": "string" + }, + "MaxResults": { + "name": "maxResults", + "in": "query", + "type": "integer", + "minimum": 1 + }, + "Prefix": { + "name": "prefix", + "in": "query", + "type": "string" + }, + "Upn": { + "name": "upn", + "in": "query", + "type": "boolean" + }, + "Action_Required": { + "name": "action", + "in": "query", + "required": true, + "type": "string", + "enum": ["append", "flush", "setAccessControl", "setAccessControlRecursive", "setProperties"] + }, + "Action_GetAccessControl": { + "name": "action", + "in": "query", + "type": "string", + "enum": ["getAccessControl", "getStatus"] + }, + "Mode": { + "name": "mode", + "in": "query", + "type": "string", + "enum": ["set", "modify", "remove"] + }, + "Position": { + "name": "position", + "in": "query", + "type": "integer", + "format": "int64" + }, + "RetainUncommittedData": { + "name": "retainUncommittedData", + "in": "query", + "type": "boolean" + }, + "Close": { + "name": "close", + "in": "query", + "type": "boolean" + }, + "MaxRecords": { + "name": "maxRecords", + "in": "query", + "type": "integer" + }, + "ContentLength": { + "name": "Content-Length", + "in": "header", + "type": "integer", + "format": "int64" + }, + "ContentMD5": { + "name": "Content-MD5", + "in": "header", + "type": "string" + }, + "Range": { + "name": "Range", + "in": "header", + "type": "string" + }, + "Properties": { + "name": "x-ms-properties", + "in": "header", + "type": "string" + }, + "Owner": { + "name": "x-ms-owner", + "in": "header", + "type": "string" + }, + "Group": { + "name": "x-ms-group", + "in": "header", + "type": "string" + }, + "Permissions": { + "name": "x-ms-permissions", + "in": "header", + "type": "string" + }, + "PermissionsOptional": { + "name": "x-ms-permissions", + "in": "header", + "type": "string" + }, + "Umask": { + "name": "x-ms-umask", + "in": "header", + "type": "string" + }, + "Acl": { + "name": "x-ms-acl", + "in": "header", + "type": "string" + }, + "RenameSource": { + "name": "x-ms-rename-source", + "in": "header", + "type": "string" + }, + "LeaseAction": { + "name": "x-ms-lease-action", + "in": "header", + "required": true, + "type": "string", + "enum": ["acquire", "release", "renew", "break", "change"] + }, + "LeaseDuration": { + "name": "x-ms-lease-duration", + "in": "header", + "type": "integer" + }, + "LeaseBreakPeriod": { + "name": "x-ms-lease-break-period", + "in": "header", + "type": "integer" + }, + "LeaseIdOptional": { + "name": "x-ms-lease-id", + "in": "header", + "type": "string" + }, + "ProposedLeaseId": { + "name": "x-ms-proposed-lease-id", + "in": "header", + "type": "string" + }, + "IfMatch": { + "name": "If-Match", + "in": "header", + "type": "string" + }, + "IfNoneMatch": { + "name": "If-None-Match", + "in": "header", + "type": "string" + }, + "IfModifiedSince": { + "name": "If-Modified-Since", + "in": "header", + "type": "string", + "format": "date-time-rfc1123" + }, + "IfUnmodifiedSince": { + "name": "If-Unmodified-Since", + "in": "header", + "type": "string", + "format": "date-time-rfc1123" + }, + "SourceIfMatch": { + "name": "x-ms-source-if-match", + "in": "header", + "type": "string" + }, + "SourceIfNoneMatch": { + "name": "x-ms-source-if-none-match", + "in": "header", + "type": "string" + }, + "SourceIfModifiedSince": { + "name": "x-ms-source-if-modified-since", + "in": "header", + "type": "string", + "format": "date-time-rfc1123" + }, + "SourceIfUnmodifiedSince": { + "name": "x-ms-source-if-unmodified-since", + "in": "header", + "type": "string", + "format": "date-time-rfc1123" + }, + "ApiVersionParameter": { + "name": "x-ms-version", + "in": "header", + "type": "string" + }, + "ClientRequestId": { + "name": "x-ms-client-request-id", + "in": "header", + "type": "string" + } + } +} diff --git a/swagger/dfs.md b/swagger/dfs.md new file mode 100644 index 000000000..f4d44b572 --- /dev/null +++ b/swagger/dfs.md @@ -0,0 +1,35 @@ +# Azurite Server DFS (ADLS Gen2) + +> see https://aka.ms/autorest + +```yaml +package-name: azurite-server-dfs +title: AzuriteServerDfs +description: Azurite Server for DFS (ADLS Gen2) +enable-xml: false +generate-metadata: false +license-header: MICROSOFT_MIT_NO_VERSION +output-folder: ../src/blob/generated-dfs +input-file: dfs-storage-2023-11-03.json +model-date-time-as-string: true +optional-response-headers: true +enum-types: true +``` + +## Notes + +The DFS API uses JSON (not XML like Blob), so `enable-xml` is false. + +The AutoRest code generator used by Azurite (`autorest.typescript.server`) is a +custom fork not publicly available. To regenerate, run: + +``` +autorest ./swagger/dfs.md --typescript --use= +``` + +## Changes Made to Standard DFS Swagger + +1. Made `x-ms-version` header optional to match emulator behavior. +2. Added `x-ms-rename-source` header to Path_Create for rename operations. +3. Made `resource` query parameter optional on Path operations (required only for create). +4. Added lease action operations as distinct operations rather than header-dispatched variants. diff --git a/tests/blob/dfsAclEnforcer.test.ts b/tests/blob/dfsAclEnforcer.test.ts new file mode 100644 index 000000000..00c05f86f --- /dev/null +++ b/tests/blob/dfsAclEnforcer.test.ts @@ -0,0 +1,172 @@ +/** + * Unit tests for DFS ACL enforcement logic. + * + * These test the pure ACL evaluation algorithm without requiring a running + * server or real JWT tokens. The enforcer is invoked by PathHandler when + * --oauth acl is enabled (Phase III). + */ + +import * as assert from "assert"; +import { + checkAcl, + parseAcl, + AclPermission +} from "../../src/blob/dfs/DfsAclEnforcer"; +import { IDfsAuthenticatedIdentity } from "../../src/blob/dfs/DfsContext"; + +describe("DFS ACL Enforcer", () => { + + describe("parseAcl", () => { + it("parses a valid ACL string", () => { + const acl = parseAcl("user::rwx,user:abc-123:r-x,group::r--,mask::rwx,other::---"); + assert.strictEqual(acl.length, 5); + + assert.strictEqual(acl[0].type, "user"); + assert.strictEqual(acl[0].entityId, ""); + assert.strictEqual(acl[0].read, true); + assert.strictEqual(acl[0].write, true); + assert.strictEqual(acl[0].execute, true); + + assert.strictEqual(acl[1].type, "user"); + assert.strictEqual(acl[1].entityId, "abc-123"); + assert.strictEqual(acl[1].read, true); + assert.strictEqual(acl[1].write, false); + assert.strictEqual(acl[1].execute, true); + + assert.strictEqual(acl[3].type, "mask"); + assert.strictEqual(acl[4].type, "other"); + assert.strictEqual(acl[4].read, false); + }); + + it("returns empty array for undefined", () => { + assert.deepStrictEqual(parseAcl(undefined), []); + }); + + it("returns empty array for empty string", () => { + assert.deepStrictEqual(parseAcl(""), []); + }); + }); + + describe("checkAcl — bypass scenarios", () => { + it("bypasses when no identity (emulator mode)", () => { + const result = checkAcl(undefined, "owner1", "group1", "rwxr-x---", undefined, "r"); + assert.strictEqual(result.allowed, true); + assert.ok(result.reason.includes("emulator mode")); + }); + + it("bypasses when identity has no oid or upn", () => { + const identity: IDfsAuthenticatedIdentity = {}; + const result = checkAcl(identity, "owner1", "group1", "rwxr-x---", undefined, "r"); + assert.strictEqual(result.allowed, true); + }); + + it("bypasses when owner is $superuser", () => { + const identity: IDfsAuthenticatedIdentity = { oid: "user1" }; + const result = checkAcl(identity, "$superuser", "$superuser", "rwxr-x---", undefined, "r"); + assert.strictEqual(result.allowed, true); + assert.ok(result.reason.includes("$superuser")); + }); + + it("bypasses when owner is undefined (defaults to $superuser)", () => { + const identity: IDfsAuthenticatedIdentity = { oid: "user1" }; + const result = checkAcl(identity, undefined, undefined, undefined, undefined, "r"); + assert.strictEqual(result.allowed, true); + }); + }); + + describe("checkAcl — owner permissions", () => { + it("allows owner with read permission", () => { + const identity: IDfsAuthenticatedIdentity = { oid: "owner1" }; + const result = checkAcl(identity, "owner1", "group1", "r-x------", undefined, "r"); + assert.strictEqual(result.allowed, true); + assert.ok(result.reason.includes("Owner")); + }); + + it("denies owner without write permission", () => { + const identity: IDfsAuthenticatedIdentity = { oid: "owner1" }; + const result = checkAcl(identity, "owner1", "group1", "r-x------", undefined, "w"); + assert.strictEqual(result.allowed, false); + }); + + it("allows owner with full rwx permissions", () => { + const identity: IDfsAuthenticatedIdentity = { oid: "owner1" }; + for (const perm of ["r", "w", "x"] as AclPermission[]) { + const result = checkAcl(identity, "owner1", "group1", "rwx------", undefined, perm); + assert.strictEqual(result.allowed, true, `Expected owner to have ${perm}`); + } + }); + }); + + describe("checkAcl — named user ACL entries", () => { + it("allows named user with matching ACL entry", () => { + const identity: IDfsAuthenticatedIdentity = { oid: "user-abc" }; + const acl = "user::rwx,user:user-abc:r-x,group::r--,other::---"; + const result = checkAcl(identity, "owner1", "group1", "rwxr-----", acl, "r"); + assert.strictEqual(result.allowed, true); + assert.ok(result.reason.includes("Named user")); + }); + + it("denies named user without required permission", () => { + const identity: IDfsAuthenticatedIdentity = { oid: "user-abc" }; + const acl = "user::rwx,user:user-abc:r--,group::r--,other::---"; + const result = checkAcl(identity, "owner1", "group1", "rwxr-----", acl, "w"); + assert.strictEqual(result.allowed, false); + }); + + it("applies mask to named user permissions", () => { + const identity: IDfsAuthenticatedIdentity = { oid: "user-abc" }; + // user has rwx but mask limits to r-- + const acl = "user::rwx,user:user-abc:rwx,mask::r--,group::r--,other::---"; + const result = checkAcl(identity, "owner1", "group1", "rwxr-----", acl, "w"); + assert.strictEqual(result.allowed, false); // mask denies write + }); + }); + + describe("checkAcl — other permissions", () => { + it("falls through to other permissions for unknown user", () => { + const identity: IDfsAuthenticatedIdentity = { oid: "stranger" }; + const result = checkAcl(identity, "owner1", "group1", "rwxr-xr--", undefined, "r"); + assert.strictEqual(result.allowed, true); + assert.ok(result.reason.includes("Other")); + }); + + it("denies stranger when other has no permissions", () => { + const identity: IDfsAuthenticatedIdentity = { oid: "stranger" }; + const result = checkAcl(identity, "owner1", "group1", "rwxr-x---", undefined, "r"); + assert.strictEqual(result.allowed, false); + }); + + it("allows stranger when other has read", () => { + const identity: IDfsAuthenticatedIdentity = { oid: "stranger" }; + const result = checkAcl(identity, "owner1", "group1", "------r--", undefined, "r"); + assert.strictEqual(result.allowed, true); + }); + }); + + describe("checkAcl — group permissions", () => { + it("allows group member with group permissions", () => { + const identity: IDfsAuthenticatedIdentity = { oid: "group1" }; + const result = checkAcl(identity, "owner1", "group1", "---rwx---", undefined, "r"); + assert.strictEqual(result.allowed, true); + assert.ok(result.reason.includes("Group")); + }); + + it("denies group member without required permission", () => { + const identity: IDfsAuthenticatedIdentity = { oid: "group1" }; + const result = checkAcl(identity, "owner1", "group1", "------r--", undefined, "r"); + // group perms are chars 3-5 = "---" → denied, falls to other = "r--" + // Actually the caller matches group so it checks group perms first + // chars 3-5 = "---" → denied + assert.strictEqual(result.allowed, false); + }); + }); + + describe("checkAcl — UPN matching", () => { + it("matches identity by upn when oid is not set", () => { + const identity: IDfsAuthenticatedIdentity = { upn: "user@example.com" }; + const result = checkAcl(identity, "user@example.com", "group1", "rwx------", undefined, "r"); + assert.strictEqual(result.allowed, true); + assert.ok(result.reason.includes("Owner")); + }); + }); +}); diff --git a/tests/blob/dfsProxy.test.ts b/tests/blob/dfsProxy.test.ts new file mode 100644 index 000000000..6609176cb --- /dev/null +++ b/tests/blob/dfsProxy.test.ts @@ -0,0 +1,818 @@ +import { + AccountSASPermissions, + AccountSASResourceTypes, + AccountSASServices, + BlobServiceClient, + generateAccountSASQueryParameters, + newPipeline, + SASProtocol, + StorageSharedKeyCredential +} from "@azure/storage-blob"; +import axios from "axios"; +import * as assert from "assert"; + +import DfsConfiguration from "../../src/blob/DfsConfiguration"; +import DfsServer from "../../src/blob/DfsServer"; +import BlobServer from "../../src/blob/BlobServer"; +import { BLOB_API_VERSION } from "../../src/blob/utils/constants"; +import { configLogger } from "../../src/common/Logger"; +import BlobTestServerFactory from "../BlobTestServerFactory"; +import { + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + getUniqueName +} from "../testutils"; + +configLogger(false); + +describe("DfsProxy", () => { + const factory = new BlobTestServerFactory(); + const blobServer = factory.createServer(); + + const dfsConfig = new DfsConfiguration( + "127.0.0.1", + 11004 + ); + const dfsServer = new DfsServer( + dfsConfig, + (blobServer as BlobServer).metadataStore, + (blobServer as BlobServer).extentStore, + (blobServer as BlobServer).accountDataStore + ); + + const blobServiceClient = new BlobServiceClient( + `http://${blobServer.config.host}:${blobServer.config.port}/${EMULATOR_ACCOUNT_NAME}`, + newPipeline(new StorageSharedKeyCredential(EMULATOR_ACCOUNT_NAME, EMULATOR_ACCOUNT_KEY), { + retryOptions: { maxTries: 1 }, + keepAliveOptions: { enable: false } + }) + ); + + const sas = generateAccountSASQueryParameters( + { + expiresOn: new Date(Date.now() + 60 * 60 * 1000), + startsOn: new Date(Date.now() - 10 * 60 * 1000), + permissions: AccountSASPermissions.parse("rwdlacupitfx"), + resourceTypes: AccountSASResourceTypes.parse("sco").toString(), + services: AccountSASServices.parse("b").toString(), + protocol: SASProtocol.HttpsAndHttp + }, + new StorageSharedKeyCredential(EMULATOR_ACCOUNT_NAME, EMULATOR_ACCOUNT_KEY) + ).toString(); + + const dfsBaseUrl = `http://${dfsConfig.host}:${dfsConfig.port}/${EMULATOR_ACCOUNT_NAME}`; + + before(async () => { + await blobServer.start(); + await dfsServer.start(); + }); + + after(async () => { + await dfsServer.close(); + await blobServer.close(); + await blobServer.clean(); + }); + + it("maps filesystem create and delete to container operations @loki @sql", async () => { + const fileSystemName = getUniqueName("fs"); + const createUrl = `${dfsBaseUrl}/${fileSystemName}?resource=filesystem&${sas}`; + + const createResponse = await axios.put(createUrl, undefined, { + headers: { + "x-ms-version": BLOB_API_VERSION + }, + validateStatus: () => true + }); + + assert.strictEqual(createResponse.status, 201); + + const created = await blobServiceClient + .getContainerClient(fileSystemName) + .getProperties(); + assert.ok(created.etag); + + const deleteResponse = await axios.delete(createUrl, { + headers: { + "x-ms-version": BLOB_API_VERSION + }, + validateStatus: () => true + }); + + assert.strictEqual(deleteResponse.status, 202); + + try { + await blobServiceClient.getContainerClient(fileSystemName).getProperties(); + assert.fail("Expected container to be deleted"); + } catch (error) { + assert.strictEqual((error as any).statusCode, 404); + } + }); + + it("maps filesystem HEAD to container properties and returns filesystem header @loki @sql", async () => { + const fileSystemName = getUniqueName("fs"); + const containerClient = blobServiceClient.getContainerClient(fileSystemName); + await containerClient.create(); + + const headUrl = `${dfsBaseUrl}/${fileSystemName}?resource=filesystem&${sas}`; + + const response = await axios.head(headUrl, { + headers: { + "x-ms-version": BLOB_API_VERSION + }, + validateStatus: () => true + }); + + assert.strictEqual(response.status, 200); + assert.strictEqual(response.headers["x-ms-resource-type"], "filesystem"); + + await containerClient.delete(); + }); + + it("creates and reads a file via DFS path operations @loki @sql", async () => { + const fileSystemName = getUniqueName("fs"); + const containerClient = blobServiceClient.getContainerClient(fileSystemName); + await containerClient.create(); + + // Create a file + const fileName = "test-file.txt"; + const createUrl = `${dfsBaseUrl}/${fileSystemName}/${fileName}?resource=file&${sas}`; + + const createResponse = await axios.put(createUrl, undefined, { + headers: { "x-ms-version": BLOB_API_VERSION }, + validateStatus: () => true + }); + assert.strictEqual(createResponse.status, 201); + + // Verify file exists via blob API + const blobClient = containerClient.getBlobClient(fileName); + const props = await blobClient.getProperties(); + assert.ok(props.etag); + assert.strictEqual(props.contentLength, 0); + + // Get path properties via DFS + const headUrl = `${dfsBaseUrl}/${fileSystemName}/${fileName}?${sas}`; + const headResponse = await axios.head(headUrl, { + headers: { "x-ms-version": BLOB_API_VERSION }, + validateStatus: () => true + }); + assert.strictEqual(headResponse.status, 200); + assert.strictEqual(headResponse.headers["x-ms-resource-type"], "file"); + + // Delete via DFS + const deleteUrl = `${dfsBaseUrl}/${fileSystemName}/${fileName}?${sas}`; + const deleteResponse = await axios.delete(deleteUrl, { + headers: { "x-ms-version": BLOB_API_VERSION }, + validateStatus: () => true + }); + assert.strictEqual(deleteResponse.status, 200); + + await containerClient.delete(); + }); + + it("creates a directory with hdi_isfolder metadata @loki @sql", async () => { + const fileSystemName = getUniqueName("fs"); + const containerClient = blobServiceClient.getContainerClient(fileSystemName); + await containerClient.create(); + + const dirName = "test-dir"; + const createUrl = `${dfsBaseUrl}/${fileSystemName}/${dirName}?resource=directory&${sas}`; + + const createResponse = await axios.put(createUrl, undefined, { + headers: { "x-ms-version": BLOB_API_VERSION }, + validateStatus: () => true + }); + assert.strictEqual(createResponse.status, 201); + + // Verify it's a directory via DFS HEAD + const headUrl = `${dfsBaseUrl}/${fileSystemName}/${dirName}?${sas}`; + const headResponse = await axios.head(headUrl, { + headers: { "x-ms-version": BLOB_API_VERSION }, + validateStatus: () => true + }); + assert.strictEqual(headResponse.status, 200); + assert.strictEqual(headResponse.headers["x-ms-resource-type"], "directory"); + + // Delete directory + const deleteUrl = `${dfsBaseUrl}/${fileSystemName}/${dirName}?recursive=true&${sas}`; + const deleteResponse = await axios.delete(deleteUrl, { + headers: { "x-ms-version": BLOB_API_VERSION }, + validateStatus: () => true + }); + assert.strictEqual(deleteResponse.status, 200); + + await containerClient.delete(); + }); + + it("lists paths in a filesystem @loki @sql", async () => { + const fileSystemName = getUniqueName("fs"); + const containerClient = blobServiceClient.getContainerClient(fileSystemName); + await containerClient.create(); + + // Create some files via DFS + for (const name of ["file1.txt", "file2.txt", "dir1"]) { + const resource = name === "dir1" ? "directory" : "file"; + const url = `${dfsBaseUrl}/${fileSystemName}/${name}?resource=${resource}&${sas}`; + await axios.put(url, undefined, { + headers: { "x-ms-version": BLOB_API_VERSION }, + validateStatus: () => true + }); + } + + // List paths + const listUrl = `${dfsBaseUrl}/${fileSystemName}?resource=filesystem&recursive=true&${sas}`; + const listResponse = await axios.get(listUrl, { + headers: { "x-ms-version": BLOB_API_VERSION }, + validateStatus: () => true + }); + + assert.strictEqual(listResponse.status, 200); + assert.ok(listResponse.data.paths); + assert.ok(listResponse.data.paths.length >= 3); + + const pathNames = listResponse.data.paths.map((p: any) => p.name); + assert.ok(pathNames.includes("file1.txt")); + assert.ok(pathNames.includes("file2.txt")); + assert.ok(pathNames.includes("dir1")); + + // Verify dir1 is marked as directory + const dir1 = listResponse.data.paths.find((p: any) => p.name === "dir1"); + assert.strictEqual(dir1.isDirectory, true); + + await containerClient.delete(); + }); + + it("appends data and flushes to create file content @loki @sql", async () => { + const fileSystemName = getUniqueName("fs"); + const containerClient = blobServiceClient.getContainerClient(fileSystemName); + await containerClient.create(); + + const fileName = "append-test.txt"; + + // Create empty file + const createUrl = `${dfsBaseUrl}/${fileSystemName}/${fileName}?resource=file&${sas}`; + await axios.put(createUrl, undefined, { + headers: { "x-ms-version": BLOB_API_VERSION }, + validateStatus: () => true + }); + + // Append data + const data1 = "Hello, "; + const data2 = "World!"; + + const append1Url = `${dfsBaseUrl}/${fileSystemName}/${fileName}?action=append&position=0&${sas}`; + const append1Response = await fetch(append1Url, { + method: "PATCH", + headers: { + "x-ms-version": BLOB_API_VERSION, + "Content-Type": "application/octet-stream" + }, + body: data1 + }); + assert.strictEqual(append1Response.status, 202); + + const append2Url = `${dfsBaseUrl}/${fileSystemName}/${fileName}?action=append&position=${Buffer.byteLength(data1)}&${sas}`; + const append2Response = await fetch(append2Url, { + method: "PATCH", + headers: { + "x-ms-version": BLOB_API_VERSION, + "Content-Type": "application/octet-stream" + }, + body: data2 + }); + assert.strictEqual(append2Response.status, 202); + + // Flush + const totalLength = Buffer.byteLength(data1) + Buffer.byteLength(data2); + const flushUrl = `${dfsBaseUrl}/${fileSystemName}/${fileName}?action=flush&position=${totalLength}&${sas}`; + const flushResponse = await fetch(flushUrl, { + method: "PATCH", + headers: { "x-ms-version": BLOB_API_VERSION } + }); + assert.strictEqual(flushResponse.status, 200); + + // Read back via DFS + const readUrl = `${dfsBaseUrl}/${fileSystemName}/${fileName}?${sas}`; + const readResponse = await fetch(readUrl, { + headers: { "x-ms-version": BLOB_API_VERSION } + }); + assert.strictEqual(readResponse.status, 200); + const readBody = await readResponse.text(); + assert.strictEqual(readBody, "Hello, World!"); + + await containerClient.delete(); + }); + + it("renames a file via DFS @loki @sql", async () => { + const fileSystemName = getUniqueName("fs"); + const containerClient = blobServiceClient.getContainerClient(fileSystemName); + await containerClient.create(); + + // Create a file + const oldName = "old-file.txt"; + const createUrl = `${dfsBaseUrl}/${fileSystemName}/${oldName}?resource=file&${sas}`; + await axios.put(createUrl, undefined, { + headers: { "x-ms-version": BLOB_API_VERSION }, + validateStatus: () => true + }); + + // Rename it + const newName = "new-file.txt"; + const renameUrl = `${dfsBaseUrl}/${fileSystemName}/${newName}?${sas}`; + const renameResponse = await axios.put(renameUrl, undefined, { + headers: { + "x-ms-version": BLOB_API_VERSION, + "x-ms-rename-source": `/${EMULATOR_ACCOUNT_NAME}/${fileSystemName}/${oldName}` + }, + validateStatus: () => true + }); + assert.strictEqual(renameResponse.status, 201); + + // Old path should not exist + const oldHeadUrl = `${dfsBaseUrl}/${fileSystemName}/${oldName}?${sas}`; + const oldHeadResponse = await axios.head(oldHeadUrl, { + headers: { "x-ms-version": BLOB_API_VERSION }, + validateStatus: () => true + }); + assert.strictEqual(oldHeadResponse.status, 404); + + // New path should exist + const newHeadUrl = `${dfsBaseUrl}/${fileSystemName}/${newName}?${sas}`; + const newHeadResponse = await axios.head(newHeadUrl, { + headers: { "x-ms-version": BLOB_API_VERSION }, + validateStatus: () => true + }); + assert.strictEqual(newHeadResponse.status, 200); + + await containerClient.delete(); + }); + + it("sets and gets ACLs on a path @loki @sql", async () => { + const fileSystemName = getUniqueName("fs"); + const containerClient = blobServiceClient.getContainerClient(fileSystemName); + await containerClient.create(); + + // Create a file + const fileName = "acl-test.txt"; + const createUrl = `${dfsBaseUrl}/${fileSystemName}/${fileName}?resource=file&${sas}`; + await axios.put(createUrl, undefined, { + headers: { "x-ms-version": BLOB_API_VERSION }, + validateStatus: () => true + }); + + // Set ACL + const setAclUrl = `${dfsBaseUrl}/${fileSystemName}/${fileName}?action=setAccessControl&${sas}`; + const setAclResponse = await fetch(setAclUrl, { + method: "PATCH", + headers: { + "x-ms-version": BLOB_API_VERSION, + "x-ms-owner": "testowner", + "x-ms-group": "testgroup", + "x-ms-permissions": "rwxr-x---", + "x-ms-acl": "user::rwx,group::r-x,other::---" + } + }); + assert.strictEqual(setAclResponse.status, 200); + + // Get ACL + const getAclUrl = `${dfsBaseUrl}/${fileSystemName}/${fileName}?action=getAccessControl&${sas}`; + const getAclResponse = await axios.head(getAclUrl, { + headers: { "x-ms-version": BLOB_API_VERSION }, + validateStatus: () => true + }); + assert.strictEqual(getAclResponse.status, 200); + assert.strictEqual(getAclResponse.headers["x-ms-owner"], "testowner"); + assert.strictEqual(getAclResponse.headers["x-ms-group"], "testgroup"); + assert.strictEqual(getAclResponse.headers["x-ms-permissions"], "rwxr-x---"); + assert.strictEqual(getAclResponse.headers["x-ms-acl"], "user::rwx,group::r-x,other::---"); + + await containerClient.delete(); + }); + + it("sets filesystem properties via PATCH @loki @sql", async () => { + const fileSystemName = getUniqueName("fs"); + const createUrl = `${dfsBaseUrl}/${fileSystemName}?resource=filesystem&${sas}`; + await axios.put(createUrl, undefined, { + headers: { "x-ms-version": BLOB_API_VERSION }, + validateStatus: () => true + }); + + // Set properties + const propValue = Buffer.from("bar").toString("base64"); + const patchUrl = `${dfsBaseUrl}/${fileSystemName}?resource=filesystem&${sas}`; + const patchResponse = await fetch(patchUrl, { + method: "PATCH", + headers: { + "x-ms-version": BLOB_API_VERSION, + "x-ms-properties": `foo=${propValue}` + } + }); + assert.strictEqual(patchResponse.status, 200); + + // Delete + await axios.delete(createUrl, { + headers: { "x-ms-version": BLOB_API_VERSION }, + validateStatus: () => true + }); + }); + + it("validates Content-MD5 on append @loki @sql", async () => { + const fileSystemName = getUniqueName("fs"); + const containerClient = blobServiceClient.getContainerClient(fileSystemName); + await containerClient.create(); + + const fileName = "md5-test.txt"; + const createUrl = `${dfsBaseUrl}/${fileSystemName}/${fileName}?resource=file&${sas}`; + await axios.put(createUrl, undefined, { + headers: { "x-ms-version": BLOB_API_VERSION }, + validateStatus: () => true + }); + + // Append with correct MD5 + const data = "test data"; + const crypto = require("crypto"); + const correctMD5 = crypto.createHash("md5").update(data).digest("base64"); + + const appendUrl = `${dfsBaseUrl}/${fileSystemName}/${fileName}?action=append&position=0&${sas}`; + const goodResponse = await fetch(appendUrl, { + method: "PATCH", + headers: { + "x-ms-version": BLOB_API_VERSION, + "Content-Type": "application/octet-stream", + "Content-MD5": correctMD5 + }, + body: data + }); + assert.strictEqual(goodResponse.status, 202); + + // Append with wrong MD5 + const appendUrl2 = `${dfsBaseUrl}/${fileSystemName}/${fileName}?action=append&position=${Buffer.byteLength(data)}&${sas}`; + const badResponse = await fetch(appendUrl2, { + method: "PATCH", + headers: { + "x-ms-version": BLOB_API_VERSION, + "Content-Type": "application/octet-stream", + "Content-MD5": "AAAAAAAAAAAAAAAAAAAAAA==" + }, + body: "more data" + }); + assert.strictEqual(badResponse.status, 400); + const errorBody = await badResponse.json() as any; + assert.strictEqual(errorBody.error.code, "Md5Mismatch"); + + await containerClient.delete(); + }); + + it("respects If-Match conditional header on getProperties @loki @sql", async () => { + const fileSystemName = getUniqueName("fs"); + const containerClient = blobServiceClient.getContainerClient(fileSystemName); + await containerClient.create(); + + const fileName = "cond-test.txt"; + const createUrl = `${dfsBaseUrl}/${fileSystemName}/${fileName}?resource=file&${sas}`; + const createResponse = await axios.put(createUrl, undefined, { + headers: { "x-ms-version": BLOB_API_VERSION }, + validateStatus: () => true + }); + assert.strictEqual(createResponse.status, 201); + const etag = createResponse.headers["etag"]; + + // Matching ETag should succeed + const headUrl = `${dfsBaseUrl}/${fileSystemName}/${fileName}?${sas}`; + const matchResponse = await axios.head(headUrl, { + headers: { + "x-ms-version": BLOB_API_VERSION, + "If-Match": etag + }, + validateStatus: () => true + }); + assert.strictEqual(matchResponse.status, 200); + + // Non-matching ETag should fail with 412 + const noMatchResponse = await axios.head(headUrl, { + headers: { + "x-ms-version": BLOB_API_VERSION, + "If-Match": `"0xDEADBEEF"` + }, + validateStatus: () => true + }); + assert.strictEqual(noMatchResponse.status, 412); + + await containerClient.delete(); + }); + + it("respects If-None-Match conditional header on read @loki @sql", async () => { + const fileSystemName = getUniqueName("fs"); + const containerClient = blobServiceClient.getContainerClient(fileSystemName); + await containerClient.create(); + + const fileName = "cond-read.txt"; + const createUrl = `${dfsBaseUrl}/${fileSystemName}/${fileName}?resource=file&${sas}`; + const createResponse = await axios.put(createUrl, undefined, { + headers: { "x-ms-version": BLOB_API_VERSION }, + validateStatus: () => true + }); + const etag = createResponse.headers["etag"]; + + // Read with non-matching If-None-Match should succeed + const readUrl = `${dfsBaseUrl}/${fileSystemName}/${fileName}?${sas}`; + const readResponse = await fetch(readUrl, { + headers: { + "x-ms-version": BLOB_API_VERSION, + "If-None-Match": `"0xDEADBEEF"` + } + }); + assert.strictEqual(readResponse.status, 200); + + // Read with matching If-None-Match should return 304 + const notModifiedResponse = await fetch(readUrl, { + headers: { + "x-ms-version": BLOB_API_VERSION, + "If-None-Match": etag + } + }); + assert.strictEqual(notModifiedResponse.status, 304); + + await containerClient.delete(); + }); + + it("acquires, renews, and releases a lease on a path @loki @sql", async () => { + const fileSystemName = getUniqueName("fs"); + const containerClient = blobServiceClient.getContainerClient(fileSystemName); + await containerClient.create(); + + const fileName = "lease-test.txt"; + const createUrl = `${dfsBaseUrl}/${fileSystemName}/${fileName}?resource=file&${sas}`; + await axios.put(createUrl, undefined, { + headers: { "x-ms-version": BLOB_API_VERSION }, + validateStatus: () => true + }); + + const pathUrl = `${dfsBaseUrl}/${fileSystemName}/${fileName}?${sas}`; + + // Acquire lease + const acquireResponse = await fetch(pathUrl, { + method: "POST", + headers: { + "x-ms-version": BLOB_API_VERSION, + "x-ms-lease-action": "acquire", + "x-ms-lease-duration": "60" + } + }); + assert.strictEqual(acquireResponse.status, 201); + const leaseId = acquireResponse.headers.get("x-ms-lease-id"); + assert.ok(leaseId); + + // Renew lease + const renewResponse = await fetch(pathUrl, { + method: "POST", + headers: { + "x-ms-version": BLOB_API_VERSION, + "x-ms-lease-action": "renew", + "x-ms-lease-id": leaseId! + } + }); + assert.strictEqual(renewResponse.status, 200); + + // Release lease + const releaseResponse = await fetch(pathUrl, { + method: "POST", + headers: { + "x-ms-version": BLOB_API_VERSION, + "x-ms-lease-action": "release", + "x-ms-lease-id": leaseId! + } + }); + assert.strictEqual(releaseResponse.status, 200); + + await containerClient.delete(); + }); + + it("breaks a lease on a path @loki @sql", async () => { + const fileSystemName = getUniqueName("fs"); + const containerClient = blobServiceClient.getContainerClient(fileSystemName); + await containerClient.create(); + + const fileName = "break-lease.txt"; + const createUrl = `${dfsBaseUrl}/${fileSystemName}/${fileName}?resource=file&${sas}`; + await axios.put(createUrl, undefined, { + headers: { "x-ms-version": BLOB_API_VERSION }, + validateStatus: () => true + }); + + const pathUrl = `${dfsBaseUrl}/${fileSystemName}/${fileName}?${sas}`; + + // Acquire lease first + const acquireResponse = await fetch(pathUrl, { + method: "POST", + headers: { + "x-ms-version": BLOB_API_VERSION, + "x-ms-lease-action": "acquire", + "x-ms-lease-duration": "60" + } + }); + assert.strictEqual(acquireResponse.status, 201); + + // Break lease + const breakResponse = await fetch(pathUrl, { + method: "POST", + headers: { + "x-ms-version": BLOB_API_VERSION, + "x-ms-lease-action": "break" + } + }); + assert.strictEqual(breakResponse.status, 202); + + await containerClient.delete(); + }); + + it("changes a lease on a path @loki @sql", async () => { + const fileSystemName = getUniqueName("fs"); + const containerClient = blobServiceClient.getContainerClient(fileSystemName); + await containerClient.create(); + + const fileName = "change-lease.txt"; + const createUrl = `${dfsBaseUrl}/${fileSystemName}/${fileName}?resource=file&${sas}`; + await axios.put(createUrl, undefined, { + headers: { "x-ms-version": BLOB_API_VERSION }, + validateStatus: () => true + }); + + const pathUrl = `${dfsBaseUrl}/${fileSystemName}/${fileName}?${sas}`; + + // Acquire lease + const acquireResponse = await fetch(pathUrl, { + method: "POST", + headers: { + "x-ms-version": BLOB_API_VERSION, + "x-ms-lease-action": "acquire", + "x-ms-lease-duration": "60" + } + }); + assert.strictEqual(acquireResponse.status, 201); + const leaseId = acquireResponse.headers.get("x-ms-lease-id"); + assert.ok(leaseId); + + // Change lease + const newLeaseId = "d7e6eb60-f905-4b44-a090-123456789012"; + const changeResponse = await fetch(pathUrl, { + method: "POST", + headers: { + "x-ms-version": BLOB_API_VERSION, + "x-ms-lease-action": "change", + "x-ms-lease-id": leaseId!, + "x-ms-proposed-lease-id": newLeaseId + } + }); + assert.strictEqual(changeResponse.status, 200); + assert.strictEqual(changeResponse.headers.get("x-ms-lease-id"), newLeaseId); + + // Release with new lease ID + await fetch(pathUrl, { + method: "POST", + headers: { + "x-ms-version": BLOB_API_VERSION, + "x-ms-lease-action": "release", + "x-ms-lease-id": newLeaseId + } + }); + + await containerClient.delete(); + }); + + it("renames a directory and its children atomically @loki @sql", async () => { + const fileSystemName = getUniqueName("fs"); + const containerClient = blobServiceClient.getContainerClient(fileSystemName); + await containerClient.create(); + + // Create a directory with children + const dirName = "src-dir"; + const createDirUrl = `${dfsBaseUrl}/${fileSystemName}/${dirName}?resource=directory&${sas}`; + await axios.put(createDirUrl, undefined, { + headers: { "x-ms-version": BLOB_API_VERSION }, + validateStatus: () => true + }); + + for (const child of ["child1.txt", "child2.txt"]) { + const createFileUrl = `${dfsBaseUrl}/${fileSystemName}/${dirName}/${child}?resource=file&${sas}`; + await axios.put(createFileUrl, undefined, { + headers: { "x-ms-version": BLOB_API_VERSION }, + validateStatus: () => true + }); + } + + // Rename directory + const newDirName = "dest-dir"; + const renameUrl = `${dfsBaseUrl}/${fileSystemName}/${newDirName}?${sas}`; + const renameResponse = await axios.put(renameUrl, undefined, { + headers: { + "x-ms-version": BLOB_API_VERSION, + "x-ms-rename-source": `/${EMULATOR_ACCOUNT_NAME}/${fileSystemName}/${dirName}` + }, + validateStatus: () => true + }); + assert.strictEqual(renameResponse.status, 201); + + // Verify old dir doesn't exist + const oldHeadUrl = `${dfsBaseUrl}/${fileSystemName}/${dirName}?${sas}`; + const oldHeadResponse = await axios.head(oldHeadUrl, { + headers: { "x-ms-version": BLOB_API_VERSION }, + validateStatus: () => true + }); + assert.strictEqual(oldHeadResponse.status, 404); + + // Verify new dir exists + const newHeadUrl = `${dfsBaseUrl}/${fileSystemName}/${newDirName}?${sas}`; + const newHeadResponse = await axios.head(newHeadUrl, { + headers: { "x-ms-version": BLOB_API_VERSION }, + validateStatus: () => true + }); + assert.strictEqual(newHeadResponse.status, 200); + assert.strictEqual(newHeadResponse.headers["x-ms-resource-type"], "directory"); + + // Verify children were moved + for (const child of ["child1.txt", "child2.txt"]) { + const childUrl = `${dfsBaseUrl}/${fileSystemName}/${newDirName}/${child}?${sas}`; + const childResponse = await axios.head(childUrl, { + headers: { "x-ms-version": BLOB_API_VERSION }, + validateStatus: () => true + }); + assert.strictEqual(childResponse.status, 200, `Expected ${newDirName}/${child} to exist`); + } + + await containerClient.delete(); + }); + + it("prevents deleting non-empty directory without recursive flag @loki @sql", async () => { + const fileSystemName = getUniqueName("fs"); + const containerClient = blobServiceClient.getContainerClient(fileSystemName); + await containerClient.create(); + + // Create directory with a child file + const dirName = "nonempty-dir"; + await axios.put( + `${dfsBaseUrl}/${fileSystemName}/${dirName}?resource=directory&${sas}`, + undefined, + { headers: { "x-ms-version": BLOB_API_VERSION }, validateStatus: () => true } + ); + await axios.put( + `${dfsBaseUrl}/${fileSystemName}/${dirName}/file.txt?resource=file&${sas}`, + undefined, + { headers: { "x-ms-version": BLOB_API_VERSION }, validateStatus: () => true } + ); + + // Try to delete without recursive — should fail with 409 + const deleteUrl = `${dfsBaseUrl}/${fileSystemName}/${dirName}?${sas}`; + const deleteResponse = await axios.delete(deleteUrl, { + headers: { "x-ms-version": BLOB_API_VERSION }, + validateStatus: () => true + }); + assert.strictEqual(deleteResponse.status, 409); + assert.strictEqual(deleteResponse.data.error.code, "DirectoryNotEmpty"); + + // Delete with recursive=true should succeed + const recursiveDeleteUrl = `${dfsBaseUrl}/${fileSystemName}/${dirName}?recursive=true&${sas}`; + const recursiveDeleteResponse = await axios.delete(recursiveDeleteUrl, { + headers: { "x-ms-version": BLOB_API_VERSION }, + validateStatus: () => true + }); + assert.strictEqual(recursiveDeleteResponse.status, 200); + + // Verify directory is gone + const headUrl = `${dfsBaseUrl}/${fileSystemName}/${dirName}?${sas}`; + const headResponse = await axios.head(headUrl, { + headers: { "x-ms-version": BLOB_API_VERSION }, + validateStatus: () => true + }); + assert.strictEqual(headResponse.status, 404); + + await containerClient.delete(); + }); + + it("auto-creates intermediate directories in HNS hierarchy @loki @sql", async () => { + const fileSystemName = getUniqueName("fs"); + const containerClient = blobServiceClient.getContainerClient(fileSystemName); + await containerClient.create(); + + // Create a deeply nested file — intermediate dirs should be created + const deepPath = "a/b/c/deep-file.txt"; + const createUrl = `${dfsBaseUrl}/${fileSystemName}/${deepPath}?resource=file&${sas}`; + const createResponse = await axios.put(createUrl, undefined, { + headers: { "x-ms-version": BLOB_API_VERSION }, + validateStatus: () => true + }); + assert.strictEqual(createResponse.status, 201); + + // Verify intermediate directories exist + for (const dir of ["a", "a/b", "a/b/c"]) { + const headUrl = `${dfsBaseUrl}/${fileSystemName}/${dir}?${sas}`; + const headResponse = await axios.head(headUrl, { + headers: { "x-ms-version": BLOB_API_VERSION }, + validateStatus: () => true + }); + assert.strictEqual(headResponse.status, 200, `Expected directory ${dir} to exist`); + assert.strictEqual(headResponse.headers["x-ms-resource-type"], "directory"); + } + + await containerClient.delete(); + }); +}); diff --git a/tests/blob/dfsSDKIntegration.test.ts b/tests/blob/dfsSDKIntegration.test.ts new file mode 100644 index 000000000..1640c98e9 --- /dev/null +++ b/tests/blob/dfsSDKIntegration.test.ts @@ -0,0 +1,483 @@ +/** + * SDK Integration Tests for ADLS Gen2 (DFS) endpoint. + * + * Uses @azure/storage-file-datalake SDK to validate that the Azurite DFS + * endpoint is compatible with the official Azure DataLake SDK. + * + * Wiki requirement: "Pass all language SDK tests" — this covers the JS SDK. + */ + +import { + DataLakeServiceClient, + DataLakeFileSystemClient, + StorageSharedKeyCredential +} from "@azure/storage-file-datalake"; +import * as assert from "assert"; + +import DfsConfiguration from "../../src/blob/DfsConfiguration"; +import DfsServer from "../../src/blob/DfsServer"; +import BlobServer from "../../src/blob/BlobServer"; +import { configLogger } from "../../src/common/Logger"; +import BlobTestServerFactory from "../BlobTestServerFactory"; +import { + EMULATOR_ACCOUNT_NAME, + getUniqueName +} from "../testutils"; + +const EMULATOR_ACCOUNT_KEY_STR = + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="; + +configLogger(false); + +describe("DFS SDK Integration (@azure/storage-file-datalake)", () => { + const factory = new BlobTestServerFactory(); + const blobServer = factory.createServer(); + + const dfsConfig = new DfsConfiguration("127.0.0.1", 11004); + const dfsServer = new DfsServer( + dfsConfig, + (blobServer as BlobServer).metadataStore, + (blobServer as BlobServer).extentStore, + (blobServer as BlobServer).accountDataStore + ); + + // The DataLake SDK connects to the DFS endpoint + const sharedKeyCredential = new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY_STR + ); + + const serviceClient = new DataLakeServiceClient( + `http://127.0.0.1:11004/${EMULATOR_ACCOUNT_NAME}`, + sharedKeyCredential + ); + + before(async () => { + await blobServer.start(); + await dfsServer.start(); + }); + + after(async () => { + await dfsServer.close(); + await blobServer.close(); + await blobServer.clean(); + }); + + // --------------------------------------------------------------------------- + // Filesystem operations + // --------------------------------------------------------------------------- + + describe("Filesystem operations", () => { + it("creates and deletes a filesystem @loki @sql", async () => { + const fsName = getUniqueName("sdkfs"); + const fsClient = serviceClient.getFileSystemClient(fsName); + + const createResponse = await fsClient.create(); + assert.strictEqual(createResponse._response.status, 201); + + const deleteResponse = await fsClient.delete(); + assert.strictEqual(deleteResponse._response.status, 202); + }); + + it("gets filesystem properties @loki @sql", async () => { + const fsName = getUniqueName("sdkfs"); + const fsClient = serviceClient.getFileSystemClient(fsName); + await fsClient.create(); + + const props = await fsClient.getProperties(); + assert.ok(props.etag); + assert.ok(props.lastModified); + + await fsClient.delete(); + }); + + it("lists filesystems @loki @sql", async () => { + const fsName = getUniqueName("sdkfs"); + const fsClient = serviceClient.getFileSystemClient(fsName); + await fsClient.create(); + + const filesystems: string[] = []; + for await (const fs of serviceClient.listFileSystems()) { + filesystems.push(fs.name); + } + assert.ok(filesystems.includes(fsName), `Expected ${fsName} in filesystem list`); + + await fsClient.delete(); + }); + }); + + // --------------------------------------------------------------------------- + // Directory operations + // --------------------------------------------------------------------------- + + describe("Directory operations", () => { + let fsClient: DataLakeFileSystemClient; + + beforeEach(async () => { + fsClient = serviceClient.getFileSystemClient(getUniqueName("sdkfs")); + await fsClient.create(); + }); + + afterEach(async () => { + await fsClient.delete(); + }); + + it("creates and deletes a directory @loki @sql", async () => { + const dirClient = fsClient.getDirectoryClient("test-dir"); + const createResponse = await dirClient.create(); + assert.strictEqual(createResponse._response.status, 201); + + const props = await dirClient.getProperties(); + assert.ok(props.etag); + + await dirClient.delete(); + }); + + it("creates nested directories @loki @sql", async () => { + const dirClient = fsClient.getDirectoryClient("parent/child/grandchild"); + await dirClient.create(); + + // Verify all intermediate dirs exist + const parentProps = await fsClient.getDirectoryClient("parent").getProperties(); + assert.ok(parentProps.etag); + + const childProps = await fsClient.getDirectoryClient("parent/child").getProperties(); + assert.ok(childProps.etag); + + const grandchildProps = await dirClient.getProperties(); + assert.ok(grandchildProps.etag); + + await fsClient.getDirectoryClient("parent").delete(true); + }); + + it("moves (renames) a directory @loki @sql", async () => { + const srcDir = fsClient.getDirectoryClient("src-dir"); + await srcDir.create(); + + // Create a file inside + const fileClient = srcDir.getFileClient("file.txt"); + await fileClient.create(); + + // Move (rename) directory + await srcDir.move("dest-dir"); + + // Verify new path exists + const destProps = await fsClient.getDirectoryClient("dest-dir").getProperties(); + assert.ok(destProps.etag); + + // Verify old path doesn't exist + try { + await fsClient.getDirectoryClient("src-dir").getProperties(); + assert.fail("Expected 404 for old directory"); + } catch (error: any) { + assert.strictEqual(error.statusCode, 404); + } + + await fsClient.getDirectoryClient("dest-dir").delete(true); + }); + }); + + // --------------------------------------------------------------------------- + // File operations + // --------------------------------------------------------------------------- + + describe("File operations", () => { + let fsClient: DataLakeFileSystemClient; + + beforeEach(async () => { + fsClient = serviceClient.getFileSystemClient(getUniqueName("sdkfs")); + await fsClient.create(); + }); + + afterEach(async () => { + await fsClient.delete(); + }); + + it("creates an empty file @loki @sql", async () => { + const fileClient = fsClient.getFileClient("empty-file.txt"); + const createResponse = await fileClient.create(); + assert.strictEqual(createResponse._response.status, 201); + + const props = await fileClient.getProperties(); + assert.ok(props.etag); + assert.strictEqual(props.contentLength, 0); + + await fileClient.delete(); + }); + + it("appends and flushes data, then reads it back @loki @sql", async () => { + const fileClient = fsClient.getFileClient("data-file.txt"); + await fileClient.create(); + + const content = "Hello from the DataLake SDK!"; + const buffer = Buffer.from(content); + + // Append + flush + await fileClient.append(buffer, 0, buffer.length); + await fileClient.flush(buffer.length); + + // Read back + const downloadResponse = await fileClient.read(); + const downloaded = await streamToString(downloadResponse.readableStreamBody!); + assert.strictEqual(downloaded, content); + + await fileClient.delete(); + }); + + it("writes multi-chunk file and reads back @loki @sql", async () => { + const fileClient = fsClient.getFileClient("multi-chunk.txt"); + await fileClient.create(); + + const chunk1 = Buffer.from("First chunk. "); + const chunk2 = Buffer.from("Second chunk. "); + const chunk3 = Buffer.from("Third chunk."); + + await fileClient.append(chunk1, 0, chunk1.length); + await fileClient.append(chunk2, chunk1.length, chunk2.length); + await fileClient.append(chunk3, chunk1.length + chunk2.length, chunk3.length); + await fileClient.flush(chunk1.length + chunk2.length + chunk3.length); + + const downloadResponse = await fileClient.read(); + const downloaded = await streamToString(downloadResponse.readableStreamBody!); + assert.strictEqual(downloaded, "First chunk. Second chunk. Third chunk."); + + await fileClient.delete(); + }); + + it("deletes a file @loki @sql", async () => { + const fileClient = fsClient.getFileClient("to-delete.txt"); + await fileClient.create(); + + await fileClient.delete(); + + try { + await fileClient.getProperties(); + assert.fail("Expected 404 after delete"); + } catch (error: any) { + assert.strictEqual(error.statusCode, 404); + } + }); + + it("moves (renames) a file @loki @sql", async () => { + const fileClient = fsClient.getFileClient("original.txt"); + await fileClient.create(); + + await fileClient.move("renamed.txt"); + + const renamedProps = await fsClient.getFileClient("renamed.txt").getProperties(); + assert.ok(renamedProps.etag); + + try { + await fsClient.getFileClient("original.txt").getProperties(); + assert.fail("Expected 404 for old file"); + } catch (error: any) { + assert.strictEqual(error.statusCode, 404); + } + + await fsClient.getFileClient("renamed.txt").delete(); + }); + }); + + // --------------------------------------------------------------------------- + // ACL operations + // --------------------------------------------------------------------------- + + describe("ACL operations", () => { + let fsClient: DataLakeFileSystemClient; + + beforeEach(async () => { + fsClient = serviceClient.getFileSystemClient(getUniqueName("sdkfs")); + await fsClient.create(); + }); + + afterEach(async () => { + await fsClient.delete(); + }); + + it("sets and gets access control on a file @loki @sql", async () => { + const fileClient = fsClient.getFileClient("acl-file.txt"); + await fileClient.create(); + + await fileClient.setAccessControl( + [ + { accessControlType: "user", defaultScope: false, entityId: "", permissions: { read: true, write: true, execute: true } }, + { accessControlType: "group", defaultScope: false, entityId: "", permissions: { read: true, write: false, execute: true } }, + { accessControlType: "other", defaultScope: false, entityId: "", permissions: { read: false, write: false, execute: false } } + ] + ); + + const acl = await fileClient.getAccessControl(); + assert.ok(acl.owner); + assert.ok(acl.group); + assert.ok(acl.permissions); + + await fileClient.delete(); + }); + + it("sets permissions on a directory @loki @sql", async () => { + const dirClient = fsClient.getDirectoryClient("acl-dir"); + await dirClient.create(); + + await dirClient.setPermissions({ + owner: { read: true, write: true, execute: true }, + group: { read: true, write: false, execute: true }, + other: { read: false, write: false, execute: false }, + stickyBit: false, + extendedAcls: false + }); + + const acl = await dirClient.getAccessControl(); + assert.ok(acl.permissions); + + await dirClient.delete(); + }); + }); + + // --------------------------------------------------------------------------- + // List paths + // --------------------------------------------------------------------------- + + describe("List paths", () => { + let fsClient: DataLakeFileSystemClient; + + beforeEach(async () => { + fsClient = serviceClient.getFileSystemClient(getUniqueName("sdkfs")); + await fsClient.create(); + }); + + afterEach(async () => { + await fsClient.delete(); + }); + + it("lists paths recursively @loki @sql", async () => { + await fsClient.getDirectoryClient("dir1").create(); + await fsClient.getFileClient("dir1/file1.txt").create(); + await fsClient.getFileClient("dir1/file2.txt").create(); + await fsClient.getFileClient("root-file.txt").create(); + + const paths: string[] = []; + for await (const path of fsClient.listPaths({ recursive: true })) { + paths.push(path.name!); + } + + assert.ok(paths.includes("dir1"), "Expected dir1 in path list"); + assert.ok(paths.includes("dir1/file1.txt"), "Expected dir1/file1.txt"); + assert.ok(paths.includes("dir1/file2.txt"), "Expected dir1/file2.txt"); + assert.ok(paths.includes("root-file.txt"), "Expected root-file.txt"); + }); + + it("lists paths non-recursively (directory level) @loki @sql", async () => { + await fsClient.getDirectoryClient("dir-a").create(); + await fsClient.getFileClient("dir-a/nested.txt").create(); + await fsClient.getFileClient("top-level.txt").create(); + + const paths: string[] = []; + for await (const path of fsClient.listPaths({ recursive: false })) { + paths.push(path.name!); + } + + assert.ok(paths.includes("dir-a"), "Expected dir-a in non-recursive list"); + assert.ok(paths.includes("top-level.txt"), "Expected top-level.txt"); + // nested file should NOT appear at top level + assert.ok(!paths.includes("dir-a/nested.txt"), "dir-a/nested.txt should not appear in non-recursive list"); + }); + }); + + // --------------------------------------------------------------------------- + // Lease operations via SDK + // --------------------------------------------------------------------------- + + describe("Lease operations", () => { + let fsClient: DataLakeFileSystemClient; + + beforeEach(async () => { + fsClient = serviceClient.getFileSystemClient(getUniqueName("sdkfs")); + await fsClient.create(); + }); + + afterEach(async () => { + await fsClient.delete(); + }); + + it("checks lease state on a file @loki @sql", async () => { + const fileClient = fsClient.getFileClient("lease-file.txt"); + await fileClient.create(); + + const props = await fileClient.getProperties(); + assert.strictEqual(props.leaseState, "available"); + + await fileClient.delete(); + }); + }); + + // --------------------------------------------------------------------------- + // Cross-API compatibility + // --------------------------------------------------------------------------- + + describe("Cross-API compatibility", () => { + let fsClient: DataLakeFileSystemClient; + let fsName: string; + + beforeEach(async () => { + fsName = getUniqueName("sdkfs"); + fsClient = serviceClient.getFileSystemClient(fsName); + await fsClient.create(); + }); + + afterEach(async () => { + await fsClient.delete(); + }); + + it("file created via DFS is visible via Blob API @loki @sql", async () => { + const fileClient = fsClient.getFileClient("cross-api-file.txt"); + await fileClient.create(); + + // Append and flush content + const content = Buffer.from("cross-api content"); + await fileClient.append(content, 0, content.length); + await fileClient.flush(content.length); + + // Read via Blob API + const { BlobServiceClient, StorageSharedKeyCredential: BlobCredential } = await import("@azure/storage-blob"); + const blobServiceClient = new BlobServiceClient( + `http://127.0.0.1:${blobServer.config.port}/${EMULATOR_ACCOUNT_NAME}`, + new BlobCredential(EMULATOR_ACCOUNT_NAME, EMULATOR_ACCOUNT_KEY_STR) + ); + const containerClient = blobServiceClient.getContainerClient(fsName); + const blobClient = containerClient.getBlobClient("cross-api-file.txt"); + const downloadResponse = await blobClient.download(); + const downloaded = await streamToString(downloadResponse.readableStreamBody!); + assert.strictEqual(downloaded, "cross-api content"); + }); + + it("blob created via Blob API is visible via DFS @loki @sql", async () => { + const { BlobServiceClient, StorageSharedKeyCredential: BlobCredential } = await import("@azure/storage-blob"); + const blobServiceClient = new BlobServiceClient( + `http://127.0.0.1:${blobServer.config.port}/${EMULATOR_ACCOUNT_NAME}`, + new BlobCredential(EMULATOR_ACCOUNT_NAME, EMULATOR_ACCOUNT_KEY_STR) + ); + const containerClient = blobServiceClient.getContainerClient(fsName); + + // Upload blob via Blob API + const content = "blob-api content"; + const blockBlobClient = containerClient.getBlockBlobClient("blob-created.txt"); + await blockBlobClient.upload(content, content.length); + + // Read via DFS + const fileClient = fsClient.getFileClient("blob-created.txt"); + const readResponse = await fileClient.read(); + const downloaded = await streamToString(readResponse.readableStreamBody!); + assert.strictEqual(downloaded, "blob-api content"); + }); + }); +}); + +// Helper to convert a readable stream to string +async function streamToString(stream: NodeJS.ReadableStream): Promise { + const chunks: Buffer[] = []; + return new Promise((resolve, reject) => { + stream.on("data", (chunk: Buffer) => chunks.push(chunk)); + stream.on("end", () => resolve(Buffer.concat(chunks).toString("utf8"))); + stream.on("error", reject); + }); +}