feat: support downloading files without extraction

.github/workflows/push.yml

@@ -53,10 +53,14 @@ jobs:
           command: yarn install --frozen-lockfile
       - name: Compile
         run: npm run build
+      - name: Unit tests
+        run: npm test
       - name: Run example
         run: cd example && node ../dist/bin.js
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Check plain file was downloaded
+        run: test -s example/downloaded/LICENSE.txt
       - name: Install Python
         uses: actions/setup-python@v5
         with:
@@ -65,3 +69,5 @@ jobs:
         run: cd example && node ../dist/bin.js
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Check plain file was downloaded
+        run: test -s example/downloaded/LICENSE.txt

.gitignore

@@ -1,4 +1,5 @@
 yarn-error.log
 .idea
 node_modules
-dist
+dist
+example/downloaded

README.md

@@ -20,6 +20,46 @@
 
 # Additional
 
+## Downloading plain files (no extraction)
+
+By default the downloaded file is extracted into the package directory. Whether a file is
+an archive is detected automatically from its mime type (resolved from the url via
+[mime-types](https://www.npmjs.com/package/mime-types)): `application/gzip`,
+`application/x-tar` and `application/zip` (`.tar.gz`, `.tgz`, `.tar`, `.zip`) are
+extracted, anything else - a `.jar`, a `.node`, a bare binary - is downloaded as is.
+
+For a file that is not extracted you must specify `dst` - a destination path relative to
+the package directory. Missing directories are created, escaping the package directory is
+not allowed.
+
+```json
+{
+    "resources": {
+        "files": [{
+            "host": "https://repo1.maven.org/maven2/",
+            "path": "org/slf4j/slf4j-api/${slf4j_version}/slf4j-api-${slf4j_version}.jar",
+            "dst": "lib/slf4j.jar"
+        }]
+    }
+}
+```
+
+`dst` supports the same variables as `host` & `path`.
+
+Detection can be overridden with `extract`:
+
+```json
+{
+    "host": "https://example.com/",
+    "path": "bundle.tar.gz",
+    "extract": false,
+    "dst": "vendor/bundle.tar.gz"
+}
+```
+
+For archives `dst` is ignored (they are always extracted into the package directory).
+
+
 ## Constraints
 
 Variables and files supports contstraints, you can define it:

example/package.json

@@ -34,6 +34,11 @@
         "host": "github_artifact://cube-js/post-installer/actions/current",
         "name": "native-${platform}-${arch}-${libc}-${libpython_version}.node",
         "not_found": "Your system is not supported by @cubejs-backend/native, some feature will be unavailable."
+      },
+      {
+        "host": "https://raw.githubusercontent.com/",
+        "path": "cube-js/post-installer/master/LICENSE",
+        "dst": "downloaded/LICENSE.txt"
       }
     ]
   }

package.json

@@ -10,7 +10,7 @@
     "build": "tsc",
     "tsc": "tsc --build",
     "tsc:watch": "tsc --build --watch",
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "vitest run"
   },
   "engines": {
     "node": ">=20.0.0"
@@ -22,13 +22,16 @@
   "license": "Apache-2.0",
   "devDependencies": {
     "@tsconfig/node20": "^20.1.6",
+    "@types/mime-types": "^3.0.1",
     "@types/node": "^20",
-    "typescript": "^5.0.4"
+    "typescript": "^5.0.4",
+    "vitest": "^3"
   },
   "dependencies": {
     "@cubejs-backend/shared": "1.7.42",
     "@octokit/core": "^5",
     "@octokit/plugin-rest-endpoint-methods": "^10",
+    "mime-types": "^3.0.2",
     "source-map-support": "^0.5.21"
   },
   "publishConfig": {