Make the database portable and encryptable (#3848)

.gitattributes

@@ -0,0 +1,8 @@
+# Vendored third-party sources. Marking them keeps them out of language
+# statistics and collapses them in diffs, so a routine engine update does not
+# read as a 400,000 line change and does not push review tooling past its
+# size limits.
+vm/ByteCodeTranslator/src/cn1_sqlite3_amalgamation.h linguist-vendored linguist-generated -diff
+vm/ByteCodeTranslator/src/cn1_sqlite3.h             linguist-vendored linguist-generated -diff
+Ports/JavaScriptPort/src/main/webapp/js/sqlite3mc.js              linguist-vendored -diff
+Ports/JavaScriptPort/src/main/webapp/js/sqlite3-opfs-async-proxy.js linguist-vendored -diff

.github/codeql/codeql-config.yml

@@ -9,3 +9,10 @@ name: "Codename One CodeQL configuration"
 # String sinks across the rest of the codebase).
 paths-ignore:
   - 'scripts/cn1playground/**/bsh/cn1/gen/**'
+  # Vendored SQLite engine (SQLite3 Multiple Ciphers), 13MB of upstream C carried
+  # verbatim. Analysing it reports upstream's findings as ours, and its sheer size
+  # pushes CodeQL off incremental analysis so it re-reports pre-existing alerts
+  # across unrelated files. Provenance and update instructions are in
+  # vm/ByteCodeTranslator/src/cn1_sqlite3_README.md.
+  - 'vm/ByteCodeTranslator/src/cn1_sqlite3_amalgamation.h'
+  - 'vm/ByteCodeTranslator/src/cn1_sqlite3.h'

.github/workflows/archetype-smoke.yml

@@ -57,14 +57,22 @@ jobs:
       - name: Install xvfb
         run: bash scripts/ci/apt-get-update.sh && sudo apt-get install -y --no-install-recommends xvfb
       - name: Build Codename One Maven artifacts (includes archetypes)
+        # Through the retry wrapper, narrowed to resolution failures: this died with
+        # "Plugin maven-install-plugin:2.5.2 or one of its dependencies could not be resolved"
+        # before it had installed anything, one of several jobs Maven Central refused on this
+        # branch in a day. A build failure in our own code still fails on the first attempt.
+        env:
+          RETRY_ONLY_MATCHING: 'status: (403|429|50[0-9])|Could not transfer artifact|Unresolveable build extension|Non-resolvable import POM|or one of its dependencies could not be resolved|authorization failed for https://repo\.maven\.apache\.org|Connection reset|Premature end of Content-Length'
+          RETRY_ATTEMPTS: '5'
+          RETRY_DELAY_SECONDS: '60'
         run: |
           # The cn1{app,lib}-archetype modules are inlined into the maven/
           # reactor since the migration, so this single install also stages
           # the archetypes into the local repo for `archetype:generate`.
           # -Darchetype.test.skip=true keeps the smoke fast — the per-tag
           # ITs already run in the release workflow.
           cd maven
-          xvfb-run -a mvn install -Plocal-dev-javase -DskipTests -Darchetype.test.skip=true -Dmaven.javadoc.skip=true
+          bash ../scripts/ci/retry.sh xvfb-run -a mvn install -Plocal-dev-javase -DskipTests -Darchetype.test.skip=true -Dmaven.javadoc.skip=true
       - name: Refresh archetype catalog
         run: |
           cd maven

.github/workflows/guibuilder.yml

@@ -100,7 +100,13 @@ jobs:
           export JAVA_HOME="${JAVA_HOME_8}"
           export PATH="${JAVA_HOME}/bin:${PATH}"
           cd scripts/guibuilder
-          xvfb-run -a mvn -B -pl javase -am -Pexecutable-jar package \
+          # Through the retry wrapper: this step resolves plugins from Maven Central, and the
+          # runner edge intermittently resets the connection mid-download, which Maven treats as a
+          # permanent resolution failure -- the packaging died on a truncated
+          # org.eclipse.sisu.plexus jar with every test already green. A real packaging failure
+          # fails the same way on each attempt, so retrying costs only time. See scripts/ci/retry.sh.
+          xvfb-run -a bash ../../scripts/ci/retry.sh \
+            mvn -B -pl javase -am -Pexecutable-jar package \
             -Dcodename1.platform=javase -Dmaven.test.skip=true
           ls javase/target/codenameone-guibuilder-*.jar
 

.github/workflows/ios-packaging.yml

@@ -237,10 +237,20 @@ jobs:
           ./scripts/run-ios-native-tests.sh \
             "${{ steps.build_ios_app.outputs.workspace }}" \
             "${{ steps.build_ios_app.outputs.scheme }}"
-        # 20 was enough while the sample built one app. It declares a watchMain now, so the app
-        # scheme carries an embedded watch app with a ParparVM translation of its own and
-        # xcodebuild builds that before running a single test. The same raise was already applied
-        # to this step in scripts-ios-native.yml; this workflow runs it too and was missed.
+        # Two things grew into this budget, and both still apply. The reuse above is partial:
+        # run-ios-native-tests.sh injects the test target into the pbxproj before building, and
+        # Xcode rebuilds the app target after a project change even though the objects are there,
+        # so this step still pays a near-full native build -- 6m55s of the 8m13s it takes on a
+        # fast runner. Measured on one commit, a slow runner spent 4m31s booting the simulator
+        # (a fast one: 1m15s) and 13 minutes on that build, and the cap fired before the tests
+        # started. On top of that the sample now declares a watchMain, so the app scheme carries
+        # an embedded watch app with a ParparVM translation of its own, which xcodebuild builds
+        # before running a single test -- the raise scripts-ios-native.yml already took.
+        #
+        # So the higher of the two, not either alone: the watch build is added work on top of the
+        # rebuild, not a re-description of it. The real fix is to inject before the UI-smoke step
+        # so this build is genuinely incremental; until then the budget has to cover the build it
+        # actually does.
         timeout-minutes: 45
 
       - name: Upload packaging artifacts

.github/workflows/javase-cef-ffmpeg-smoke.yml

@@ -79,7 +79,7 @@ jobs:
           # confirm the tool it installed is how a transient upstream outage becomes an
           # unreadable failure in somebody else's test.
           $ok = $false
-          foreach ($delay in 0, 30, 90) {
+          foreach ($delay in 0, 30, 90, 180, 300) {
             if ($delay -gt 0) {
               Write-Host "choco install ffmpeg failed; retrying in $delay s..."
               Start-Sleep -Seconds $delay

.github/workflows/linux-build-run.yml

@@ -88,9 +88,14 @@ jobs:
         with: { distribution: 'temurin', java-version: '8' }
 
       - name: Build plugin + cn1-ads-mock (JDK 8)
+        # Maven Central intermittently serves 403/429 from the runner CDN edge and Maven treats it
+        # as a permanent resolution failure, which killed this job at dependency download before
+        # anything was compiled. See scripts/ci/retry.sh; these steps only build, so a blanket
+        # retry cannot turn a flaky test into a pass.
         run: |
           cd maven
-          JAVA_HOME="$JDK_8_HOME" mvn -B -pl codenameone-maven-plugin,cn1-ads-mock -am \
+          JAVA_HOME="$JDK_8_HOME" bash ../scripts/ci/retry.sh \
+            mvn -B -pl codenameone-maven-plugin,cn1-ads-mock -am \
             -DskipTests '-Dmaven.javadoc.skip=true' '-Plocal-dev-javase' install
 
       - name: Build hellocodenameone-common (JDK 17)
@@ -102,7 +107,8 @@ jobs:
           sleep 2
           mkdir -p "$HOME/.codenameone"
           [ -f "$HOME/.codenameone/guibuilder.jar" ] || : > "$HOME/.codenameone/guibuilder.jar"
-          mvn -B -f scripts/hellocodenameone/pom.xml -pl common -am install -DskipTests '-Dmaven.javadoc.skip=true'
+          bash scripts/ci/retry.sh \
+            mvn -B -f scripts/hellocodenameone/pom.xml -pl common -am install -DskipTests '-Dmaven.javadoc.skip=true'
 
       - name: Package suite classes
         run: |
@@ -181,9 +187,23 @@ jobs:
       # plugin -- so this step works on arm64 too. The suite classes the translator
       # also needs come from the prepare-suite artifact below.
       - name: Build core + Linux port (JDK 8)
+        # Retried for the same reason as the two steps above, which this one was left out of:
+        # Maven Central answers 429 from the runner CDN edge and Maven treats it as a permanent
+        # resolution failure, so the job dies at plugin download with nothing built. See
+        # scripts/ci/retry.sh; this step only builds, so a blanket retry cannot turn a flaky test
+        # into a pass.
+        #
+        # Five attempts a minute apart rather than the default three at thirty seconds: this failed
+        # with "authorization failed for https://repo.maven.apache.org" on all three, one of
+        # several jobs Central refused on this branch in a day, and a window of about a minute is
+        # shorter than those refusals have lasted.
+        env:
+          RETRY_ATTEMPTS: '5'
+          RETRY_DELAY_SECONDS: '60'
         run: |
           cd maven
-          JAVA_HOME="$JDK_8_HOME" mvn -B -pl linux -am \
+          JAVA_HOME="$JDK_8_HOME" bash ../scripts/ci/retry.sh \
+            mvn -B -pl linux -am \
             -DskipTests '-Dmaven.javadoc.skip=true' '-Plocal-dev-javase' install
 
       - name: Download suite classes
@@ -265,7 +285,7 @@ jobs:
           echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope >/dev/null 2>&1 || true
           mkdir -p /tmp/cn1-cores
           rc=0
-          mvn -B clean package -pl JavaAPI -am -DskipTests
+          bash "$GITHUB_WORKSPACE/scripts/ci/retry.sh" mvn -B clean package -pl JavaAPI -am -DskipTests
           mvn -B test -pl tests -am \
             '-Dtest=CleanTargetLinuxIntegrationTest#capturesHelloSuiteOverWebSocketLinux' \
             '-Dsurefire.failIfNoSpecifiedTests=false' || rc=$?

.github/workflows/parparvm-tests-windows.yml

@@ -210,8 +210,13 @@ jobs:
           # than with a function and splatting, because that shape is already proven
           # on these runners and this is not the place to find out about a quoting
           # difference.
+          # Five attempts out to five minutes rather than three out to ninety seconds: a run
+          # that took all three still ended on "status code: 429, reason phrase: Too Many
+          # Requests" two minutes after it started, which is a rate limit that had not lifted
+          # yet rather than a blip. This step only downloads and builds, so waiting longer costs
+          # time and cannot turn a failing test green.
           $ok = $false
-          foreach ($delay in 0, 30, 90) {
+          foreach ($delay in 0, 30, 90, 180, 300) {
             if ($delay -gt 0) {
               Write-Host "mvn failed; retrying in $delay s..."
               Start-Sleep -Seconds $delay
@@ -226,7 +231,7 @@ jobs:
           # a gate is for, and worse than the flake it was hiding.
           $resolutionFailure = 'status: (403|429|50[0-9])|Could not transfer artifact|Unresolveable build extension|Non-resolvable import POM|Could not resolve dependencies'
           $ok = $false
-          foreach ($delay in 0, 30, 90) {
+          foreach ($delay in 0, 30, 90, 180, 300) {
             if ($delay -gt 0) {
               Write-Host "mvn failed on a dependency-resolution error; retrying in $delay s..."
               Start-Sleep -Seconds $delay

.github/workflows/parparvm-tests.yml

@@ -111,11 +111,18 @@ jobs:
         # Maven Central intermittently serves 403/429 from the runner CDN edge
         # and Maven treats it as a permanent resolution failure (killed this job
         # at dependency download, before any test ran). See scripts/ci/retry.sh.
+        #
+        # Narrowed with RETRY_ONLY_MATCHING because the second command RUNS THE
+        # TESTS: without it the wrapper retries every nonzero exit, so a test that
+        # failed once and passed on the retry would take this gate green and hide
+        # the race that produced it. The Windows counterpart restricts it for the
+        # same reason.
         run: |
           retry() { bash ../scripts/ci/retry.sh "$@"; }
           retry mvn -B clean package -pl JavaAPI -am -DskipTests
           retry mvn -B test -pl tests -am -DexcludedGroups=benchmark
         env:
+          RETRY_ONLY_MATCHING: 'status: (403|429|50[0-9])|Could not transfer artifact|Unresolveable build extension|Non-resolvable import POM|Connection reset|Premature end of Content-Length'
           JDK_8_HOME: ${{ env.JDK_8_HOME }}
           JDK_11_HOME: ${{ env.JDK_11_HOME }}
           JDK_17_HOME: ${{ env.JDK_17_HOME }}