diff --git a/.dockerignore b/.dockerignore index a7ff961b4..0d2292965 100644 --- a/.dockerignore +++ b/.dockerignore @@ -19,9 +19,8 @@ doc/ datasets/ tmp/ -# Build artifacts — rebuilt inside the image by `rake wasm:jruby_setup` +# Build artifact — rebuilt inside the image by `rake wasm:jruby_setup` lib/rbs/wasm/*.wasm -lib/rbs/wasm/jars/ *.gem ext/**/*.o ext/**/*.so diff --git a/.github/workflows/jruby.yml b/.github/workflows/jruby.yml index c9131094f..8b4dbbaf1 100644 --- a/.github/workflows/jruby.yml +++ b/.github/workflows/jruby.yml @@ -53,7 +53,7 @@ jobs: mkdir -p "$HOME/wasi-sdk" curl -sSL "$url" | tar xz --strip-components=1 -C "$HOME/wasi-sdk" echo "WASI_SDK_PATH=$HOME/wasi-sdk" >> "$GITHUB_ENV" - - name: Assemble the JRuby runtime (rbs_parser.wasm + Chicory jars) + - name: Build rbs_parser.wasm run: bundle exec rake wasm:jruby_setup - name: Set up JRuby @@ -63,5 +63,10 @@ jobs: bundler: none - name: Install runtime and test gems run: gem install prism rake rake-compiler test-unit rdoc rspec minitest json-schema pry --no-document + # jar-dependencies resolves through the JVM, so download the Chicory/ASM + # jars into ~/.m2 here on JRuby (jar-dependencies is not available in the + # CRuby step above). + - name: Download the Chicory and ASM jars + run: jruby -S rake wasm:install_jars - name: Run the test suite on JRuby run: jruby -S rake test diff --git a/.gitignore b/.gitignore index 93c5b4a5c..9ab87e2db 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,7 @@ rust/ruby-rbs/vendor/rbs/ # Compiled WebAssembly module (built by rake wasm:build) wasm/*.wasm -# JRuby runtime artifacts (assembled by rake wasm:jruby_setup, bundled in the JRuby gem) +# JRuby runtime artifact (built by rake wasm:jruby_setup, bundled in the -java +# gem). The require_jar file lib/rbs_jars.rb is committed; the Chicory/ASM jars +# themselves live in ~/.m2 (jar-dependencies), not here. lib/rbs/wasm/*.wasm -lib/rbs/wasm/jars/ diff --git a/Dockerfile.jruby b/Dockerfile.jruby index 95715cffb..8d23516bd 100644 --- a/Dockerfile.jruby +++ b/Dockerfile.jruby @@ -3,8 +3,8 @@ # RBS can't load its MRI C extension on JRuby, so it parses through a # WebAssembly build of the parser (see lib/rbs/wasm and docs/wasm_serialization.md). # This single-stage image installs the WASI SDK, compiles rbs_parser.wasm and -# vendors the Chicory/ASM jars into lib/rbs/wasm/, then runs the test suite on -# JRuby. It mirrors .github/workflows/jruby.yml but is fully self-contained. +# downloads the Chicory/ASM jars into ~/.m2 (via jar-dependencies), then runs the +# test suite on JRuby. It mirrors .github/workflows/jruby.yml but is self-contained. # # docker build -f Dockerfile.jruby -t rbs-jruby . # docker run --rm rbs-jruby # run the test suite @@ -46,9 +46,8 @@ RUN gem install prism rake rake-compiler test-unit rdoc rspec minitest json-sche WORKDIR /rbs COPY . . -# Assemble the JRuby runtime: compile rbs_parser.wasm and download the -# Chicory + ASM jars into lib/rbs/wasm/. Runs on JRuby (clang is a subprocess, -# so the build is engine independent). -RUN rake wasm:jruby_setup +# Compile rbs_parser.wasm (clang is a subprocess, so the build is engine +# independent) and download the Chicory + ASM jars into ~/.m2 via jar-dependencies. +RUN rake wasm:jruby_setup wasm:install_jars CMD ["rake", "test"] diff --git a/Rakefile b/Rakefile index c5b2aacac..bc1f34f0a 100644 --- a/Rakefile +++ b/Rakefile @@ -627,40 +627,27 @@ namespace :wasm do end end - # Where the runtime looks for the module and jars by default (see - # RBS::WASM::Runtime). These are build artifacts, bundled into the JRuby gem. + # Where the runtime looks for the module by default (see RBS::WASM::Runtime). JRUBY_WASM_DIR = File.expand_path("lib/rbs/wasm", __dir__) - CHICORY_VERSION = ENV.fetch("CHICORY_VERSION", "1.7.5") - # `compiler` is Chicory's AOT compiler (wasm -> JVM bytecode); the asm* jars - # are the ow2 ASM libraries it depends on. Keep ASM_VERSION in sync with what - # the pinned Chicory release declares. - CHICORY_JARS = %w[wasm runtime log wasi compiler].freeze - ASM_VERSION = ENV.fetch("ASM_VERSION", "9.9.1") - ASM_JARS = %w[asm asm-tree asm-util asm-commons asm-analysis].freeze - - desc "Download the Chicory and ASM jars the JRuby runtime needs into lib/rbs/wasm/jars" - task :vendor_jars do - require "open-uri" - require "fileutils" - - jars_dir = File.join(JRUBY_WASM_DIR, "jars") - FileUtils.mkdir_p(jars_dir) - - downloads = CHICORY_JARS.map { |name| ["#{name}.jar", "https://repo1.maven.org/maven2/com/dylibso/chicory/#{name}/#{CHICORY_VERSION}/#{name}-#{CHICORY_VERSION}.jar"] } - downloads += ASM_JARS.map { |name| ["#{name}.jar", "https://repo1.maven.org/maven2/org/ow2/asm/#{name}/#{ASM_VERSION}/#{name}-#{ASM_VERSION}.jar"] } - - downloads.each do |filename, url| - puts "Downloading #{url}" - URI.open(url) { |io| File.binwrite(File.join(jars_dir, filename), io.read) } # steep:ignore - end - puts "Vendored Chicory #{CHICORY_VERSION} + ASM #{ASM_VERSION} into #{jars_dir}" + desc "Download the Chicory/ASM jars into the local Maven repository (~/.m2). Run on JRuby." + task :install_jars do + # Resolves the `jar` requirements from rbs.gemspec via Maven and downloads + # them (and their transitive deps) into ~/.m2, the same way `gem install` + # does; the jars are not copied into the gem. The platform is forced to java + # because Jars::Installer skips non-java gems, and write_require_file is false + # because lib/rbs_jars.rb is hand-maintained (the generator mangles the + # `com.dylibso.chicory:runtime` artifact id). + require "jars/installer" + spec = Gem::Specification.load("rbs.gemspec") + spec.platform = "java" + Jars::Installer.new(spec).install_jars(write_require_file: false) end - desc "Assemble everything the JRuby gem needs: the .wasm and the Chicory jars" - task :jruby_setup => [:build, :vendor_jars] do + desc "Build rbs_parser.wasm and copy it next to RBS::WASM::Runtime" + task :jruby_setup => [:build] do cp WASM_OUTPUT, File.join(JRUBY_WASM_DIR, "rbs_parser.wasm") - puts "JRuby runtime is ready under #{JRUBY_WASM_DIR}" + puts "rbs_parser.wasm is ready under #{JRUBY_WASM_DIR}" end end diff --git a/Steepfile b/Steepfile index d1bc69f40..e9267ee99 100644 --- a/Steepfile +++ b/Steepfile @@ -13,6 +13,8 @@ target :lib do "lib/rbs/wasm/location.rb", "lib/rbs/wasm/runtime.rb", "lib/rbs/wasm/parser.rb", + # Generated by `rake wasm:install_jars` (jar-dependencies require_jar calls). + "lib/rbs_jars.rb", ) library "pathname", "json", "logger", "monitor", "tsort", "uri", 'dbm', 'pstore', 'singleton', 'shellwords', 'fileutils', 'find', 'digest', 'prettyprint', 'yaml', "psych", "securerandom" diff --git a/docs/release.md b/docs/release.md index 459195828..ced17b022 100644 --- a/docs/release.md +++ b/docs/release.md @@ -7,9 +7,10 @@ Each release ships **two gems**: | `rbs-X.Y.Z.gem` | `ruby` (MRI) | C extension | `rake release` (re-builds it) | | `rbs-X.Y.Z-java.gem` | `java` (JRuby) | WebAssembly (`lib/rbs/wasm`) | Docker image, pushed manually | -The `-java` gem contains no native code — just `rbs_parser.wasm` plus the -Chicory/ASM jars — so it can be built once in any environment and runs on every -JRuby. +The `-java` gem contains no native code — just `rbs_parser.wasm`. The Chicory/ASM +jars it needs are not shipped in the gem; they are declared as `jar-dependencies` +requirements and fetched from Maven when the gem is installed. So the gem can be +built once in any environment and runs on every JRuby. ## Prerequisites @@ -44,7 +45,8 @@ The `java` gem is not built by `rake release`, so build and push it manually: # Build from the committed state (the gemspec's file list comes from `git ls-files`). $ docker build -f Dockerfile.jruby -t rbs-jruby . -# Assemble rbs_parser.wasm + jars and build the -java gem into ./pkg on the host. +# Build rbs_parser.wasm and the -java gem into ./pkg on the host. The Chicory/ASM +# jars are not bundled; they are fetched from Maven when the gem is installed. $ docker run --rm -e RBS_PLATFORM=java -v "$PWD/pkg:/out" rbs-jruby \ gem build rbs.gemspec -o /out/rbs-X.Y.Z-java.gem diff --git a/lib/rbs/wasm/runtime.rb b/lib/rbs/wasm/runtime.rb index 5065b85e4..40859c9eb 100644 --- a/lib/rbs/wasm/runtime.rb +++ b/lib/rbs/wasm/runtime.rb @@ -10,21 +10,12 @@ module WASM # source string into the module's linear memory, runs the parser, and returns # the serialized result for RBS::WASM::Deserializer to rebuild. # - # Chicory is a pure-Java runtime, so there is no native dependency: only the - # `.wasm` and the Chicory jars need to ship with the gem. + # Chicory is a pure-Java runtime, so there is no native dependency. The + # `.wasm` ships in the gem; the Chicory jars are fetched from Maven by + # jar-dependencies (see lib/rbs_jars.rb and rbs.gemspec). class Runtime include MonitorMixin - # The Chicory jars the runtime needs at load time. - # Jars Chicory needs to load and run the module. - JARS = %w[wasm runtime log wasi].freeze - - # Jars for Chicory's ahead-of-time compiler (wasm -> JVM bytecode), which - # runs the parser ~8x faster than the interpreter. Optional: the runtime - # falls back to the interpreter when they are absent. asm* are the ow2 ASM - # libraries the compiler depends on. - OPTIONAL_JARS = %w[compiler asm asm-tree asm-util asm-commons asm-analysis].freeze - class << self def instance @instance ||= new @@ -33,15 +24,14 @@ def instance def wasm_path ENV["RBS_WASM_PARSER"] || File.expand_path("rbs_parser.wasm", __dir__) end - - def jars_dir - ENV["RBS_WASM_JARS"] || File.expand_path("jars", __dir__) - end end def initialize super() - load_jars + # rbs_jars.rb require_jars the Chicory/ASM jars from the local Maven + # repository (~/.m2), where jar-dependencies puts them at gem install (or + # `rake wasm:install_jars` when running from source). + require "rbs_jars" @wasm = build_instance @memory = @wasm.memory @alloc = @wasm.export("rbs_wasm_alloc") @@ -201,17 +191,6 @@ def machine_factory(wasm_module) nil end - def load_jars - JARS.each { |name| require jar_path(name) } - OPTIONAL_JARS.each do |name| - path = jar_path(name) - require path if File.exist?(path) - end - end - - def jar_path(name) - File.join(self.class.jars_dir, "#{name}.jar") - end end end end diff --git a/lib/rbs_jars.rb b/lib/rbs_jars.rb new file mode 100644 index 000000000..d0c0a8f2d --- /dev/null +++ b/lib/rbs_jars.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true +# +# Loads the Chicory/ASM jars for the JRuby WebAssembly parser (RBS::WASM::Runtime) +# from the local Maven repository. Hand-maintained, NOT auto-generated: the +# jar-dependencies generator mangles the `com.dylibso.chicory:runtime` artifact id +# into `jar` (its artifact name collides with a Maven scope keyword). Leaving out +# the usual "this is a generated file" marker also stops jar-dependencies from +# overwriting this file at gem install. Keep this list in sync with the `jar` +# requirements in rbs.gemspec. +require "jar_dependencies" + +require_jar "com.dylibso.chicory", "wasm", "1.7.5" +require_jar "com.dylibso.chicory", "runtime", "1.7.5" +require_jar "com.dylibso.chicory", "log", "1.7.5" +require_jar "com.dylibso.chicory", "wasi", "1.7.5" +require_jar "com.dylibso.chicory", "compiler", "1.7.5" +require_jar "org.ow2.asm", "asm", "9.9.1" +require_jar "org.ow2.asm", "asm-tree", "9.9.1" +require_jar "org.ow2.asm", "asm-util", "9.9.1" +require_jar "org.ow2.asm", "asm-commons", "9.9.1" +require_jar "org.ow2.asm", "asm-analysis", "9.9.1" diff --git a/rbs.gemspec b/rbs.gemspec index ee78e0426..f155c616b 100644 --- a/rbs.gemspec +++ b/rbs.gemspec @@ -38,8 +38,10 @@ Gem::Specification.new do |spec| # JRuby cannot load the MRI C extension. On JRuby (and in the `java` platform # gem, built with RBS_PLATFORM=java) RBS runs the WebAssembly-backed parser, so - # ship the prebuilt parser and the Chicory jars (assembled by - # `rake wasm:jruby_setup`) and skip the C extension. + # ship the prebuilt parser (built by `rake wasm:build`) and skip the C + # extension. The Chicory/ASM jars the runtime needs are NOT shipped in the gem: + # they are declared as `jar-dependencies` requirements below and fetched from + # Maven Central at install time (see lib/rbs/wasm/jars.rb). building_java_gem = ENV["RBS_PLATFORM"] == "java" on_jruby = defined?(RUBY_ENGINE) && RUBY_ENGINE == "jruby" @@ -47,9 +49,26 @@ Gem::Specification.new do |spec| # Only stamp the platform when building the release gem; leave it unset for # local development on JRuby so it still matches a `ruby` platform lockfile. spec.platform = "java" if building_java_gem + # rbs_parser.wasm is a build artifact (not tracked in git), so add it + # explicitly. lib/rbs_jars.rb is committed, so git ls-files already has it. spec.files += Dir.chdir(File.expand_path('..', __FILE__)) do - Dir.glob("lib/rbs/wasm/rbs_parser.wasm") + Dir.glob("lib/rbs/wasm/jars/*.jar") + Dir.glob("lib/rbs/wasm/rbs_parser.wasm") end + + # jar-dependencies (bundled with JRuby) downloads these jars from Maven when + # the gem is installed, keeping the gem small and avoiding conflicting copies. + # lib/rbs_jars.rb require_jars them at runtime; keep the two lists in sync. + spec.add_dependency "jar-dependencies", ">= 0.1.7" + spec.requirements << "jar com.dylibso.chicory:wasm, 1.7.5" + spec.requirements << "jar com.dylibso.chicory:runtime, 1.7.5" + spec.requirements << "jar com.dylibso.chicory:log, 1.7.5" + spec.requirements << "jar com.dylibso.chicory:wasi, 1.7.5" + spec.requirements << "jar com.dylibso.chicory:compiler, 1.7.5" + spec.requirements << "jar org.ow2.asm:asm, 9.9.1" + spec.requirements << "jar org.ow2.asm:asm-tree, 9.9.1" + spec.requirements << "jar org.ow2.asm:asm-util, 9.9.1" + spec.requirements << "jar org.ow2.asm:asm-commons, 9.9.1" + spec.requirements << "jar org.ow2.asm:asm-analysis, 9.9.1" else spec.extensions = %w{ext/rbs_extension/extconf.rb} end diff --git a/wasm/README.md b/wasm/README.md index 71e513a63..89a652cfd 100644 --- a/wasm/README.md +++ b/wasm/README.md @@ -19,9 +19,10 @@ The build needs the [WASI SDK](https://github.com/WebAssembly/wasi-sdk/releases) ```console $ export WASI_SDK_PATH=/path/to/wasi-sdk -$ rake wasm:build # compile rbs_parser.wasm -$ rake wasm:check # also smoke-test it (needs wasmtime) -$ rake wasm:jruby_setup # assemble lib/rbs/wasm/ for JRuby (wasm + Chicory jars) +$ rake wasm:build # compile rbs_parser.wasm +$ rake wasm:check # also smoke-test it (needs wasmtime) +$ rake wasm:jruby_setup # copy rbs_parser.wasm into lib/rbs/wasm/ for JRuby +$ rake wasm:install_jars # download the Chicory/ASM jars into ~/.m2 (run on JRuby) ``` The compiled `rbs_parser.wasm` is a build artifact and is not checked in.