From cd7da61ad948caec20a5cd63c9d9c14ae12c1a90 Mon Sep 17 00:00:00 2001 From: Garen Torikian Date: Wed, 6 May 2015 22:44:43 -0700 Subject: [PATCH 01/15] Add cmark submodule --- .gitmodules | 3 +++ ext/commonmarker/cmark | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 ext/commonmarker/cmark diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..1964f17 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "ext/commonmarker/cmark"] + path = ext/commonmarker/cmark + url = https://github.com/jgm/cmark.git diff --git a/ext/commonmarker/cmark b/ext/commonmarker/cmark new file mode 160000 index 0000000..c06c705 --- /dev/null +++ b/ext/commonmarker/cmark @@ -0,0 +1 @@ +Subproject commit c06c705260a6681a8bb5eebecd35422e388cab9f From a9e113c06772db46d5af6662034a5020cbf93e9f Mon Sep 17 00:00:00 2001 From: Garen Torikian Date: Wed, 6 May 2015 22:51:02 -0700 Subject: [PATCH 02/15] Add script to update gems + submodule --- script/bootstrap | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100755 script/bootstrap diff --git a/script/bootstrap b/script/bootstrap new file mode 100755 index 0000000..878a09d --- /dev/null +++ b/script/bootstrap @@ -0,0 +1,11 @@ +#!/bin/bash + +set -e + +echo "==> Initing Git submodules" + +git submodule update --init --recursive + +echo "==> Installing gem dependencies…" + +bundle install --path vendor/gems --local --standalone --clean "$@" From 944a8b6b47394f11a0e22991df06fa391489db63 Mon Sep 17 00:00:00 2001 From: Garen Torikian Date: Wed, 6 May 2015 22:51:08 -0700 Subject: [PATCH 03/15] Ignore things that need ignoring --- .gitignore | 3 ++- commonmarker.gemspec | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index f2c1360..7f7c50b 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ /test/tmp/ /test/version_tmp/ /tmp/ +/vendor/ +Gemfile.lock ## Specific to RubyMotion: .dat* @@ -26,7 +28,6 @@ build/ # for a library or gem, you might want to ignore these files since the code is # intended to run in multiple environments; otherwise, check them in: -# Gemfile.lock # .ruby-version # .ruby-gemset diff --git a/commonmarker.gemspec b/commonmarker.gemspec index 6cab70a..988fe01 100644 --- a/commonmarker.gemspec +++ b/commonmarker.gemspec @@ -31,7 +31,7 @@ Gem::Specification.new do |s| s.add_runtime_dependency "ffi", "~> 1.9.0" - s.add_development_dependency "rake-compiler", "~> 0.8.3" - s.add_development_dependency "bundler", "~> 1.7.7" + s.add_development_dependency "rake-compiler", "~> 0.9" + s.add_development_dependency "bundler", "~> 1.9" s.add_development_dependency "json", "~> 1.8.1" end From b7dcd32144fb3c3d6d737740bfac75b776a5c632 Mon Sep 17 00:00:00 2001 From: Garen Torikian Date: Thu, 7 May 2015 00:35:14 -0700 Subject: [PATCH 04/15] Ignore modifications to submodule --- .gitmodules | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitmodules b/.gitmodules index 1964f17..a8c4fb0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,4 @@ [submodule "ext/commonmarker/cmark"] path = ext/commonmarker/cmark url = https://github.com/jgm/cmark.git + ignore = dirty From 358f4ef30553e920367fc96c3f619626416c6cdb Mon Sep 17 00:00:00 2001 From: Garen Torikian Date: Thu, 7 May 2015 00:36:08 -0700 Subject: [PATCH 05/15] Produce a C bundle for the Ruby gem --- .gitignore | 1 + Rakefile | 6 +++++- ext/commonmarker/commonmarker.c | 17 +++++++++++++++++ ext/commonmarker/commonmarker.h | 11 +++++++++++ ext/commonmarker/extconf.rb | 18 ++++++++++++++++++ 5 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 ext/commonmarker/commonmarker.c create mode 100644 ext/commonmarker/commonmarker.h create mode 100644 ext/commonmarker/extconf.rb diff --git a/.gitignore b/.gitignore index 7f7c50b..daa35b2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.gem *.rbc +*.bundle /.config /coverage/ /InstalledFiles diff --git a/Rakefile b/Rakefile index cadfa46..30d2a1c 100644 --- a/Rakefile +++ b/Rakefile @@ -1,5 +1,6 @@ require 'date' require 'rake/clean' +require 'rake/extensiontask' require 'digest/md5' task :default => [:test] @@ -7,6 +8,9 @@ task :default => [:test] # Gem Spec gem_spec = Gem::Specification.load('commonmarker.gemspec') +# Ruby Extension +Rake::ExtensionTask.new('commonmarker', gem_spec) + # Packaging require 'bundler/gem_tasks' @@ -21,7 +25,7 @@ Rake::TestTask.new('test:unit') do |t| t.warning = false end -task 'test:unit' +task 'test:unit' => :compile desc 'Run unit and conformance tests' task :test => %w[test:unit] diff --git a/ext/commonmarker/commonmarker.c b/ext/commonmarker/commonmarker.c new file mode 100644 index 0000000..9c93cf5 --- /dev/null +++ b/ext/commonmarker/commonmarker.c @@ -0,0 +1,17 @@ +#include "commonmarker.h" +#include "cmark.h" + +VALUE rb_mCommonMark; + +static VALUE +rb_markdown_to_html(VALUE text) +{ + return rb_str_new2((char *)cmark_markdown_to_html((char *)RSTRING_PTR(text), RSTRING_LEN(text), 0)); +} + +__attribute__((visibility("default"))) +void Init_commonmarker() +{ + rb_mCommonMark = rb_define_module("CommonMark"); + rb_define_singleton_method(rb_mCommonMark, "markdown_to_html", rb_markdown_to_html, 1); +} diff --git a/ext/commonmarker/commonmarker.h b/ext/commonmarker/commonmarker.h new file mode 100644 index 0000000..45ff116 --- /dev/null +++ b/ext/commonmarker/commonmarker.h @@ -0,0 +1,11 @@ +#ifndef COMMONMARKER_H +#define COMMONMARKER_H + +#include "ruby.h" +#include "cmark.h" + +#define CSTR2SYM(s) (ID2SYM(rb_intern((s)))) + +void Init_commonmarker(); + +#endif diff --git a/ext/commonmarker/extconf.rb b/ext/commonmarker/extconf.rb new file mode 100644 index 0000000..a8170f0 --- /dev/null +++ b/ext/commonmarker/extconf.rb @@ -0,0 +1,18 @@ +require 'mkmf' +require 'fileutils' +require 'rbconfig' +host_os = RbConfig::CONFIG['host_os'] + +CMARK_DIR = File.expand_path(File.join(File.dirname(__FILE__), 'cmark')) +CMARK_BUILD_DIR = File.join(CMARK_DIR, 'build') +FileUtils.rm_rf(CMARK_BUILD_DIR) if File.exist?(CMARK_BUILD_DIR) +FileUtils.mkdir_p(CMARK_BUILD_DIR) + +Dir.chdir(CMARK_BUILD_DIR) do + system 'cmake ..' + system 'make' +end + +$LOCAL_LIBS << "#{CMARK_BUILD_DIR}/src/libcmark.a" + +create_makefile('commonmarker/commonmarker') From dfeb0417665d9f909a5d63ecce63837516e64abf Mon Sep 17 00:00:00 2001 From: Garen Torikian Date: Thu, 7 May 2015 00:38:44 -0700 Subject: [PATCH 06/15] Indicate extension in gemspec --- commonmarker.gemspec | 1 + 1 file changed, 1 insertion(+) diff --git a/commonmarker.gemspec b/commonmarker.gemspec index 988fe01..67e8e4a 100644 --- a/commonmarker.gemspec +++ b/commonmarker.gemspec @@ -23,6 +23,7 @@ Gem::Specification.new do |s| test/test_basics.rb test/test_pathological_inputs.rb ] + spec.extensions = ['ext/commonmarker/extconf.rb'] # = MANIFEST = s.test_files = s.files.grep(%r{^test/}) s.extra_rdoc_files = ["LICENSE"] From 7f8d2445fadbfe16322a398bcbaea0291a212cce Mon Sep 17 00:00:00 2001 From: Garen Torikian Date: Thu, 7 May 2015 00:41:08 -0700 Subject: [PATCH 07/15] Bring all the files over into the gemspec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This’ll help with distribution, once this thing is on RubyGems --- commonmarker.gemspec | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/commonmarker.gemspec b/commonmarker.gemspec index 67e8e4a..a2b4958 100644 --- a/commonmarker.gemspec +++ b/commonmarker.gemspec @@ -11,24 +11,17 @@ Gem::Specification.new do |s| s.license = 'BSD3' s.required_ruby_version = '>= 1.9.2' # = MANIFEST = - s.files = %w[ - LICENSE - Gemfile - README.md - Rakefile - commonmarker.gemspec - bin/commonmarker - lib/commonmarker.rb - test/benchmark.rb - test/test_basics.rb - test/test_pathological_inputs.rb - ] + spec.files = %w(LICENSE README.md Rakefile commonmarker.gemspec Gemfile bin/commonmarker) + spec.files += Dir.glob('lib/**/*.rb') + spec.test_files = Dir.glob('test/**/*') + spec.files += Dir.glob('ext/**/*') + spec.extensions = ['ext/commonmarker/extconf.rb'] # = MANIFEST = s.test_files = s.files.grep(%r{^test/}) s.extra_rdoc_files = ["LICENSE"] s.executables = ["commonmarker"] - s.require_paths = ["lib"] + s.require_paths = %w(lib ext) s.add_runtime_dependency "ffi", "~> 1.9.0" From eedac3664d15b8912a6d04b91e2aad0aa63e2d1c Mon Sep 17 00:00:00 2001 From: Garen Torikian Date: Thu, 7 May 2015 00:46:33 -0700 Subject: [PATCH 08/15] Define VERSION in a separate location DRY this up --- commonmarker.gemspec | 16 +++++++++------- lib/commonmarker.rb | 2 +- lib/commonmarker/version.rb | 3 +++ 3 files changed, 13 insertions(+), 8 deletions(-) create mode 100644 lib/commonmarker/version.rb diff --git a/commonmarker.gemspec b/commonmarker.gemspec index a2b4958..d673402 100644 --- a/commonmarker.gemspec +++ b/commonmarker.gemspec @@ -1,7 +1,10 @@ # encoding: utf-8 +lib = File.expand_path('../lib', __FILE__) +$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) +require 'commonmarker/version' Gem::Specification.new do |s| s.name = 'commonmarker' - s.version = '0.1' + s.version = CommonMarker::VERSION s.summary = "CommonMark parser and renderer" s.description = "A fast, safe, extensible parser for CommonMark" s.date = '2014-11-25' @@ -11,12 +14,11 @@ Gem::Specification.new do |s| s.license = 'BSD3' s.required_ruby_version = '>= 1.9.2' # = MANIFEST = - spec.files = %w(LICENSE README.md Rakefile commonmarker.gemspec Gemfile bin/commonmarker) - spec.files += Dir.glob('lib/**/*.rb') - spec.test_files = Dir.glob('test/**/*') - spec.files += Dir.glob('ext/**/*') - - spec.extensions = ['ext/commonmarker/extconf.rb'] + s.files = %w(LICENSE README.md Rakefile commonmarker.gemspec Gemfile bin/commonmarker) + s.files += Dir.glob('lib/**/*.rb') + s.files += Dir.glob('ext/**/*') + s.test_files = Dir.glob('test/**/*') + s.extensions = ['ext/commonmarker/extconf.rb'] # = MANIFEST = s.test_files = s.files.grep(%r{^test/}) s.extra_rdoc_files = ["LICENSE"] diff --git a/lib/commonmarker.rb b/lib/commonmarker.rb index 3e42df1..94f195a 100755 --- a/lib/commonmarker.rb +++ b/lib/commonmarker.rb @@ -378,7 +378,7 @@ def out(*args) end elsif arg.kind_of?(Node) self.render(arg) - else + else @stream.write(arg) end end diff --git a/lib/commonmarker/version.rb b/lib/commonmarker/version.rb new file mode 100644 index 0000000..34d6395 --- /dev/null +++ b/lib/commonmarker/version.rb @@ -0,0 +1,3 @@ +module CommonMarker + VERSION = '0.1.0' +end From c1f3aa3db69e537bbd66eba0ef467921e59bee80 Mon Sep 17 00:00:00 2001 From: Garen Torikian Date: Thu, 7 May 2015 13:21:08 -0700 Subject: [PATCH 09/15] Trash lib-ffi MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We’ll use rake-compiler for everything, which is pretty common --- Rakefile | 4 ++- commonmarker.gemspec | 1 - lib/commonmarker.rb | 86 ++++++++++++++++++-------------------------- 3 files changed, 38 insertions(+), 53 deletions(-) diff --git a/Rakefile b/Rakefile index 30d2a1c..3f45633 100644 --- a/Rakefile +++ b/Rakefile @@ -9,7 +9,9 @@ task :default => [:test] gem_spec = Gem::Specification.load('commonmarker.gemspec') # Ruby Extension -Rake::ExtensionTask.new('commonmarker', gem_spec) +Rake::ExtensionTask.new('commonmarker', gem_spec) do |ext| + ext.lib_dir = File.join('lib', 'commonmarker') +end # Packaging require 'bundler/gem_tasks' diff --git a/commonmarker.gemspec b/commonmarker.gemspec index d673402..80cd8ab 100644 --- a/commonmarker.gemspec +++ b/commonmarker.gemspec @@ -25,7 +25,6 @@ Gem::Specification.new do |s| s.executables = ["commonmarker"] s.require_paths = %w(lib ext) - s.add_runtime_dependency "ffi", "~> 1.9.0" s.add_development_dependency "rake-compiler", "~> 0.9" s.add_development_dependency "bundler", "~> 1.9" diff --git a/lib/commonmarker.rb b/lib/commonmarker.rb index 94f195a..027f243 100755 --- a/lib/commonmarker.rb +++ b/lib/commonmarker.rb @@ -1,60 +1,44 @@ #!/usr/bin/env ruby -require 'ffi' require 'stringio' require 'cgi' require 'set' require 'uri' -module CMark - extend FFI::Library - # override attach_function so ruby names don't start with cmark_ - def self.attach_function(c_name, args, returns) - ruby_name = c_name.to_s.sub(/cmark_/, "") - super(ruby_name, c_name, args, returns) - end - - ffi_lib ['libcmark', 'cmark'] - typedef :pointer, :node - enum :node_type, [:document, :blockquote, :list, :list_item, - :code_block, :html, :paragraph, - :header, :hrule, :reference_def, - :text, :softbreak, :linebreak, :code, :inline_html, - :emph, :strong, :link, :image] - enum :list_type, [:no_list, :bullet_list, :ordered_list] - - attach_function :cmark_node_new, [:node_type], :node - attach_function :cmark_free_nodes, [:node], :void - attach_function :cmark_node_unlink, [:node], :void - attach_function :cmark_node_insert_before, [:node, :node], :int - attach_function :cmark_node_insert_after, [:node, :node], :int - attach_function :cmark_node_prepend_child, [:node, :node], :int - attach_function :cmark_node_append_child, [:node, :node], :int - attach_function :cmark_markdown_to_html, [:string, :int], :string - attach_function :cmark_render_html, [:node], :string - attach_function :cmark_parse_document, [:string, :int], :node - attach_function :cmark_node_first_child, [:node], :node - attach_function :cmark_node_last_child, [:node], :node - attach_function :cmark_node_parent, [:node], :node - attach_function :cmark_node_next, [:node], :node - attach_function :cmark_node_previous, [:node], :node - attach_function :cmark_node_get_type, [:node], :node_type - attach_function :cmark_node_get_string_content, [:node], :string - attach_function :cmark_node_set_string_content, [:node, :string], :int - attach_function :cmark_node_get_url, [:node], :string - attach_function :cmark_node_set_url, [:node, :string], :int - attach_function :cmark_node_get_title, [:node], :string - attach_function :cmark_node_set_title, [:node, :string], :int - attach_function :cmark_node_get_header_level, [:node], :int - attach_function :cmark_node_set_header_level, [:node, :int], :int - attach_function :cmark_node_get_list_type, [:node], :list_type - attach_function :cmark_node_set_list_type, [:node, :list_type], :int - attach_function :cmark_node_get_list_start, [:node], :int - attach_function :cmark_node_set_list_start, [:node, :int], :int - attach_function :cmark_node_get_list_tight, [:node], :bool - attach_function :cmark_node_set_list_tight, [:node, :bool], :int - attach_function :cmark_node_get_fence_info, [:node], :string - attach_function :cmark_node_set_fence_info, [:node, :string], :int -end +# module CMark + + # attach_function :cmark_node_new, [:node_type], :node + # attach_function :cmark_node_free, [:node], :void + # attach_function :cmark_node_unlink, [:node], :void + # attach_function :cmark_node_insert_before, [:node, :node], :int + # attach_function :cmark_node_insert_after, [:node, :node], :int + # attach_function :cmark_node_prepend_child, [:node, :node], :int + # attach_function :cmark_node_append_child, [:node, :node], :int + # attach_function :cmark_markdown_to_html, [:string, :int], :string + # attach_function :cmark_render_html, [:node], :string + # attach_function :cmark_parse_document, [:string, :int], :node + # attach_function :cmark_node_first_child, [:node], :node + # attach_function :cmark_node_last_child, [:node], :node + # attach_function :cmark_node_parent, [:node], :node + # attach_function :cmark_node_next, [:node], :node + # attach_function :cmark_node_previous, [:node], :node + # attach_function :cmark_node_get_type, [:node], :node_type + # attach_function :cmark_node_get_literal, [:node], :string + # attach_function :cmark_node_set_literal, [:node, :string], :int + # attach_function :cmark_node_get_url, [:node], :string + # attach_function :cmark_node_set_url, [:node, :string], :int + # attach_function :cmark_node_get_title, [:node], :string + # attach_function :cmark_node_set_title, [:node, :string], :int + # attach_function :cmark_node_get_header_level, [:node], :int + # attach_function :cmark_node_set_header_level, [:node, :int], :int + # attach_function :cmark_node_get_list_type, [:node], :list_type + # attach_function :cmark_node_set_list_type, [:node, :list_type], :int + # attach_function :cmark_node_get_list_start, [:node], :int + # attach_function :cmark_node_set_list_start, [:node, :int], :int + # attach_function :cmark_node_get_list_tight, [:node], :bool + # attach_function :cmark_node_set_list_tight, [:node, :bool], :int + # attach_function :cmark_node_get_fence_info, [:node], :string + # attach_function :cmark_node_set_fence_info, [:node, :string], :int +# end module CommonMarker VERSION = 0.1 From a4e0e123454b954f8fa29f914a0419d1ffc904dc Mon Sep 17 00:00:00 2001 From: Garen Torikian Date: Thu, 7 May 2015 13:21:20 -0700 Subject: [PATCH 10/15] Add script to run a single test --- script/single_test | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100755 script/single_test diff --git a/script/single_test b/script/single_test new file mode 100755 index 0000000..a639741 --- /dev/null +++ b/script/single_test @@ -0,0 +1,6 @@ +#!/bin/sh + +set -e + +# bundle exec rake compile +ruby -I"lib:test" test/$1 From e2e59020994292ea9987c7c140b26282b4782cc4 Mon Sep 17 00:00:00 2001 From: Garen Torikian Date: Thu, 7 May 2015 13:23:36 -0700 Subject: [PATCH 11/15] Simplify test setup --- test/test_basics.rb | 6 +----- test/test_helper.rb | 5 +++++ test/test_pathological_inputs.rb | 3 +-- test/test_spec.rb | 6 +----- 4 files changed, 8 insertions(+), 12 deletions(-) create mode 100644 test/test_helper.rb diff --git a/test/test_basics.rb b/test/test_basics.rb index 29a450b..648f46c 100644 --- a/test/test_basics.rb +++ b/test/test_basics.rb @@ -1,6 +1,4 @@ -require 'commonmarker' -require 'minitest/autorun' -include CommonMarker +require 'test_helper' class TestNode < Minitest::Unit::TestCase def setup @@ -44,5 +42,3 @@ def teardown @doc.free end end - - diff --git a/test/test_helper.rb b/test/test_helper.rb new file mode 100644 index 0000000..97f0509 --- /dev/null +++ b/test/test_helper.rb @@ -0,0 +1,5 @@ +require 'bundler/setup' +require 'commonmarker' +require 'minitest/autorun' + +include CommonMarker diff --git a/test/test_pathological_inputs.rb b/test/test_pathological_inputs.rb index 62c5a58..d757137 100644 --- a/test/test_pathological_inputs.rb +++ b/test/test_pathological_inputs.rb @@ -1,7 +1,6 @@ # coding: UTF-8 -require 'commonmarker' +require 'test_helper' require 'minitest/benchmark' -include CommonMarker def markdown(s) Node.parse_string(s).to_html diff --git a/test/test_spec.rb b/test/test_spec.rb index e3945eb..bca1a4f 100644 --- a/test/test_spec.rb +++ b/test/test_spec.rb @@ -1,7 +1,5 @@ -require 'commonmarker' -require 'minitest/autorun' +require 'test_helper' require 'json' -include CommonMarker class TestSpec < Minitest::Unit::TestCase cases = JSON.parse(open("test/spec_tests.json", 'r').read) @@ -20,5 +18,3 @@ class TestSpec < Minitest::Unit::TestCase end end end - - From 1858c19b065e7b8047f247168bb541863876b17c Mon Sep 17 00:00:00 2001 From: Garen Torikian Date: Thu, 7 May 2015 13:23:54 -0700 Subject: [PATCH 12/15] Start creating nodes and documents --- commonmarker.gemspec | 2 +- ext/commonmarker/commonmarker.c | 38 ++++++++++++++++++++++++++++++++- ext/commonmarker/extconf.rb | 2 +- lib/commonmarker.rb | 25 ++++++++++++++++------ lib/commonmarker/config.rb | 14 ++++++++++++ test/test_maliciousness.rb | 9 ++++++++ 6 files changed, 81 insertions(+), 9 deletions(-) create mode 100644 lib/commonmarker/config.rb create mode 100644 test/test_maliciousness.rb diff --git a/commonmarker.gemspec b/commonmarker.gemspec index 80cd8ab..03f2614 100644 --- a/commonmarker.gemspec +++ b/commonmarker.gemspec @@ -25,7 +25,7 @@ Gem::Specification.new do |s| s.executables = ["commonmarker"] s.require_paths = %w(lib ext) - + s.add_dependency 'ruby-enum', '~> 0.4' s.add_development_dependency "rake-compiler", "~> 0.9" s.add_development_dependency "bundler", "~> 1.9" s.add_development_dependency "json", "~> 1.8.1" diff --git a/ext/commonmarker/commonmarker.c b/ext/commonmarker/commonmarker.c index 9c93cf5..72f999c 100644 --- a/ext/commonmarker/commonmarker.c +++ b/ext/commonmarker/commonmarker.c @@ -1,7 +1,9 @@ #include "commonmarker.h" #include "cmark.h" +#include "node.h" VALUE rb_mCommonMark; +cmark_node *node; static VALUE rb_markdown_to_html(VALUE text) @@ -9,9 +11,43 @@ rb_markdown_to_html(VALUE text) return rb_str_new2((char *)cmark_markdown_to_html((char *)RSTRING_PTR(text), RSTRING_LEN(text), 0)); } +static VALUE +rb_node_new(VALUE self, VALUE rb_type) +{ + Check_Type(rb_type, T_FIXNUM); + cmark_node_type node_type = (cmark_node_type) FIX2INT(rb_type); + + return cmark_node_new(node_type); +} + +static VALUE +rb_parse_document(VALUE self, VALUE rb_text, VALUE rb_len, VALUE rb_options) +{ + Check_Type(rb_text, T_STRING); + Check_Type(rb_len, T_FIXNUM); + Check_Type(rb_options, T_FIXNUM); + + char *text = (char *)RSTRING_PTR(rb_text); + int len = FIX2INT(rb_len); + int options = FIX2INT(rb_options); + + node = cmark_parse_document(text, len, CMARK_OPT_DEFAULT); + + return Data_Wrap_Struct(self, NULL, cmark_node_free, node); +} + +static VALUE +rb_node_get_string_content(VALUE self) +{ + return Data_Wrap_Struct(self, NULL, cmark_strbuf_free, &node->string_content); +} + __attribute__((visibility("default"))) void Init_commonmarker() { - rb_mCommonMark = rb_define_module("CommonMark"); + rb_mCommonMark = rb_define_class("CMark", rb_cObject); rb_define_singleton_method(rb_mCommonMark, "markdown_to_html", rb_markdown_to_html, 1); + rb_define_singleton_method(rb_mCommonMark, "node_new", rb_node_new, 1); + rb_define_singleton_method(rb_mCommonMark, "parse_document", rb_parse_document, 3); + rb_define_singleton_method(rb_mCommonMark, "node_get_string_content", rb_node_get_string_content, 1); } diff --git a/ext/commonmarker/extconf.rb b/ext/commonmarker/extconf.rb index a8170f0..9274624 100644 --- a/ext/commonmarker/extconf.rb +++ b/ext/commonmarker/extconf.rb @@ -5,7 +5,6 @@ CMARK_DIR = File.expand_path(File.join(File.dirname(__FILE__), 'cmark')) CMARK_BUILD_DIR = File.join(CMARK_DIR, 'build') -FileUtils.rm_rf(CMARK_BUILD_DIR) if File.exist?(CMARK_BUILD_DIR) FileUtils.mkdir_p(CMARK_BUILD_DIR) Dir.chdir(CMARK_BUILD_DIR) do @@ -13,6 +12,7 @@ system 'make' end +$CFLAGS << " -I#{CMARK_DIR}/src -I#{CMARK_BUILD_DIR}/src" $LOCAL_LIBS << "#{CMARK_BUILD_DIR}/src/libcmark.a" create_makefile('commonmarker/commonmarker') diff --git a/lib/commonmarker.rb b/lib/commonmarker.rb index 027f243..1b208d1 100755 --- a/lib/commonmarker.rb +++ b/lib/commonmarker.rb @@ -1,9 +1,18 @@ #!/usr/bin/env ruby +require 'commonmarker/commonmarker' +require 'commonmarker/config' require 'stringio' require 'cgi' require 'set' require 'uri' +NODE_TYPES = [:document, :blockquote, :list, :list_item, + :code_block, :html, :paragraph, + :header, :hrule, :reference_def, + :text, :softbreak, :linebreak, :code, :inline_html, + :emph, :strong, :link, :image] +LIST_TYPES = [:no_list, :bullet_list, :ordered_list] + # module CMark # attach_function :cmark_node_new, [:node_type], :node @@ -41,8 +50,6 @@ # end module CommonMarker - VERSION = 0.1 - class NodeError < StandardError end @@ -60,10 +67,13 @@ def initialize(type=nil, pointer=nil) if pointer @pointer = pointer else + unless NODE_TYPES.include?(type) + raise NodeError, "node type does not exist #{type}" + end @pointer = CMark.node_new(type) end - if @pointer.null? - raise NodeError, "could not create node of type " + type.to_s + if @pointer.nil? + raise NodeError, "could not create node of type #{type}" end end @@ -72,8 +82,11 @@ def initialize(type=nil, pointer=nil) # memory when it is no longer needed. # Params: # +s+:: +String+ to be parsed. - def self.parse_string(s) - Node.new(nil, CMark.parse_document(s, s.bytesize)) + def self.parse_string(s, option=:default) + unless Config.keys.include?(option) + raise StandardError, "option type does not exist #{option}" + end + Node.new(nil, CMark.parse_document(s, s.bytesize, Config.to_h[option])) end # Parses a file into a :document Node. The diff --git a/lib/commonmarker/config.rb b/lib/commonmarker/config.rb new file mode 100644 index 0000000..e50ddd0 --- /dev/null +++ b/lib/commonmarker/config.rb @@ -0,0 +1,14 @@ +require 'ruby-enum' + +module CommonMarker + class Config + include Ruby::Enum + + define :default, 0 + define :sourcepos, 1 + define :hardbreaks, 2 + define :normalize, 4 + define :smart, 8 + + end +end diff --git a/test/test_maliciousness.rb b/test/test_maliciousness.rb new file mode 100644 index 0000000..28131ea --- /dev/null +++ b/test/test_maliciousness.rb @@ -0,0 +1,9 @@ +require 'test_helper' + +class CommonMarker::TestMaliciousness < Minitest::Unit::TestCase + def test_init + assert_raises NodeError do + render = Node.new(99999) + end + end +end From 89c3606810a1011b3b169e3b0d8fd4e53adb0eb3 Mon Sep 17 00:00:00 2001 From: Garen Torikian Date: Fri, 8 May 2015 00:23:48 -0700 Subject: [PATCH 13/15] Clean up node types in Ruby --- lib/commonmarker.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/commonmarker.rb b/lib/commonmarker.rb index 1b208d1..c25a3c3 100755 --- a/lib/commonmarker.rb +++ b/lib/commonmarker.rb @@ -8,8 +8,8 @@ NODE_TYPES = [:document, :blockquote, :list, :list_item, :code_block, :html, :paragraph, - :header, :hrule, :reference_def, - :text, :softbreak, :linebreak, :code, :inline_html, + :header, :hrule, :text, :softbreak, + :linebreak, :code, :inline_html, :emph, :strong, :link, :image] LIST_TYPES = [:no_list, :bullet_list, :ordered_list] @@ -338,7 +338,7 @@ def walk(&blk) # Returns the type of this Node. def type - CMark.node_get_type(@pointer) + NODE_TYPES[CMark.node_get_type(@pointer)] end # Convert to HTML using libcmark's fast (but uncustomizable) renderer. From d2ff4202513021f52b6936a155fda468865e4d2a Mon Sep 17 00:00:00 2001 From: Garen Torikian Date: Fri, 8 May 2015 00:24:02 -0700 Subject: [PATCH 14/15] Uncomment compilation in single test --- script/single_test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/single_test b/script/single_test index a639741..a14d9d1 100755 --- a/script/single_test +++ b/script/single_test @@ -2,5 +2,5 @@ set -e -# bundle exec rake compile +bundle exec rake compile ruby -I"lib:test" test/$1 From a6fbd23822e96972dfb5903ef42cbaffbef730ff Mon Sep 17 00:00:00 2001 From: Garen Torikian Date: Sat, 9 May 2015 12:46:53 -0700 Subject: [PATCH 15/15] Support walking down the document --- ext/commonmarker/commonmarker.c | 80 +++++++++++++++++++++++++++++++-- lib/commonmarker.rb | 9 +++- 2 files changed, 83 insertions(+), 6 deletions(-) diff --git a/ext/commonmarker/commonmarker.c b/ext/commonmarker/commonmarker.c index 72f999c..dfdf9a0 100644 --- a/ext/commonmarker/commonmarker.c +++ b/ext/commonmarker/commonmarker.c @@ -3,7 +3,6 @@ #include "node.h" VALUE rb_mCommonMark; -cmark_node *node; static VALUE rb_markdown_to_html(VALUE text) @@ -31,17 +30,84 @@ rb_parse_document(VALUE self, VALUE rb_text, VALUE rb_len, VALUE rb_options) int len = FIX2INT(rb_len); int options = FIX2INT(rb_options); - node = cmark_parse_document(text, len, CMARK_OPT_DEFAULT); + cmark_node *doc = cmark_parse_document(text, len, options); - return Data_Wrap_Struct(self, NULL, cmark_node_free, node); + return Data_Wrap_Struct(self, NULL, cmark_node_free, doc); } static VALUE -rb_node_get_string_content(VALUE self) +rb_node_get_string_content(VALUE self, VALUE n) { + cmark_node *node; + Data_Get_Struct(n, cmark_node, node); + return Data_Wrap_Struct(self, NULL, cmark_strbuf_free, &node->string_content); } +static VALUE +rb_node_get_type(VALUE self, VALUE n) +{ + cmark_node *node; + Data_Get_Struct(n, cmark_node, node); + + return INT2NUM(cmark_node_get_type(node)); +} + +static VALUE +rb_node_get_type_string(VALUE self, VALUE n) +{ + cmark_node *node; + Data_Get_Struct(n, cmark_node, node); + + return rb_str_new2(cmark_node_get_type_string(node)); +} + +void +rb_node_unlink(VALUE self, VALUE n) +{ + cmark_node *node; + Data_Get_Struct(n, cmark_node, node); + + cmark_node_unlink(node); +} + +void +rb_free_nodes(VALUE self, VALUE n) +{ + cmark_node *node; + Data_Get_Struct(n, cmark_node, node); + + cmark_node_free(node); +} + +static VALUE +rb_node_first_child(VALUE self, VALUE n) +{ + cmark_node *node; + Data_Get_Struct(n, cmark_node, node); + + if (node == NULL) + return Qnil; + + cmark_node *child = cmark_node_first_child(node); + + return Data_Wrap_Struct(self, NULL, NULL, child); +} + +static VALUE +rb_node_next(VALUE self, VALUE n) +{ + cmark_node *node; + Data_Get_Struct(n, cmark_node, node); + + if (node == NULL) + return Qnil; + + cmark_node *next = cmark_node_next(node); + + return Data_Wrap_Struct(self, NULL, NULL, next); +} + __attribute__((visibility("default"))) void Init_commonmarker() { @@ -50,4 +116,10 @@ void Init_commonmarker() rb_define_singleton_method(rb_mCommonMark, "node_new", rb_node_new, 1); rb_define_singleton_method(rb_mCommonMark, "parse_document", rb_parse_document, 3); rb_define_singleton_method(rb_mCommonMark, "node_get_string_content", rb_node_get_string_content, 1); + rb_define_singleton_method(rb_mCommonMark, "node_get_type", rb_node_get_type, 1); + rb_define_singleton_method(rb_mCommonMark, "node_get_type_string", rb_node_get_type_string, 1); + rb_define_singleton_method(rb_mCommonMark, "node_unlink", rb_node_unlink, 1); + rb_define_singleton_method(rb_mCommonMark, "free_nodes", rb_free_nodes, 1); + rb_define_singleton_method(rb_mCommonMark, "node_first_child", rb_node_first_child, 1); + rb_define_singleton_method(rb_mCommonMark, "node_next", rb_node_next, 1); } diff --git a/lib/commonmarker.rb b/lib/commonmarker.rb index c25a3c3..b58334d 100755 --- a/lib/commonmarker.rb +++ b/lib/commonmarker.rb @@ -6,7 +6,7 @@ require 'set' require 'uri' -NODE_TYPES = [:document, :blockquote, :list, :list_item, +NODE_TYPES = [:none, :document, :blockquote, :list, :list_item, :code_block, :html, :paragraph, :header, :hrule, :text, :softbreak, :linebreak, :code, :inline_html, @@ -111,7 +111,7 @@ def last_child # Iterator over the children (if any) of this Node. def each_child childptr = CMark.node_first_child(@pointer) - while not childptr.null? do + until CMark.node_get_type_string(childptr) == "NONE" do nextptr = CMark.node_next(childptr) yield Node.new(nil, childptr) childptr = nextptr @@ -341,6 +341,11 @@ def type NODE_TYPES[CMark.node_get_type(@pointer)] end + def type_string + CMark.node_get_type_string(@pointer) + end + + # Convert to HTML using libcmark's fast (but uncustomizable) renderer. def to_html CMark.render_html(@pointer).force_encoding("utf-8")