Merge pull request #29 from Dahie/add-mediawiki

Add mediawiki

Author: Dahie

.github/workflows/main.yml

@@ -6,11 +6,11 @@ jobs:
   test:
     runs-on: ubuntu-latest
 
-
     steps:
     - uses: actions/checkout@v3
-    # - name: Set up Redis
-    #   uses: zhulik/redis-action@1.1.0
+    - uses: r-lib/actions/setup-pandoc@v2
+      with:
+        pandoc-version: '3.1.11' 
     - name: Install Ruby and gems
       uses: ruby/setup-ruby@v1
       with:

.rubocop.yml

@@ -1,6 +1,9 @@
+inherit_from: .rubocop_todo.yml
+
 require:
   - rubocop-rake
   - rubocop-rspec
+  - rubocop-performance
 
 AllCops:
   NewCops: enable

.rubocop_todo.yml

@@ -0,0 +1,55 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2024-03-03 10:06:52 UTC using RuboCop version 1.60.2.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 1
+# Configuration parameters: AllowedMethods, AllowedPatterns.
+Lint/NestedMethodDefinition:
+  Exclude:
+    - 'lib/caramelize/filters/textile_to_markdown.rb'
+
+# Offense count: 2
+# Configuration parameters: AllowedMethods, AllowedPatterns, CountRepeatedAttributes.
+Metrics/AbcSize:
+  Max: 79
+
+# Offense count: 1
+# Configuration parameters: CountComments, CountAsOne.
+Metrics/ClassLength:
+  Max: 118
+
+# Offense count: 1
+# Configuration parameters: AllowedMethods, AllowedPatterns.
+Metrics/CyclomaticComplexity:
+  Max: 9
+
+# Offense count: 2
+# Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns.
+Metrics/MethodLength:
+  Max: 102
+
+# Offense count: 1
+# Configuration parameters: AllowedMethods, AllowedPatterns.
+Metrics/PerceivedComplexity:
+  Max: 9
+
+# Offense count: 2
+# Configuration parameters: Include, CustomTransform, IgnoreMethods, SpecSuffixOnly.
+# Include: **/*_spec*rb*, **/spec/**/*
+RSpec/FilePath:
+  Exclude:
+    - 'spec/lib/caramelize/filters/add_newline_to_page_end_spec.rb'
+    - 'spec/lib/caramelize/filters/mediawiki_to_markdown_spec.rb'
+
+# Offense count: 2
+# Configuration parameters: Include, CustomTransform, IgnoreMethods, IgnoreMetadata.
+# Include: **/*_spec.rb
+RSpec/SpecFilePathFormat:
+  Exclude:
+    - '**/spec/routing/**/*'
+    - 'spec/lib/caramelize/filters/add_newline_to_page_end_spec.rb'
+    - 'spec/lib/caramelize/filters/mediawiki_to_markdown_spec.rb'

Gemfile

@@ -12,5 +12,6 @@ gem 'guard-rspec'
 gem 'rake'
 gem 'rspec'
 gem 'rubocop'
+gem 'rubocop-performance'
 gem 'rubocop-rake'
 gem 'rubocop-rspec'

Gemfile.lock

@@ -1,10 +1,11 @@
 PATH
   remote: .
   specs:
-    caramelize (1.2.1)
+    caramelize (1.3.0)
       commander
       gollum-lib
       mysql2
+      paru
       ruby-progressbar
 
 GEM
@@ -50,7 +51,7 @@ GEM
     highline (3.0.1)
     json (2.7.1)
     language_server-protocol (3.17.0.3)
-    listen (3.8.0)
+    listen (3.9.0)
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
     loofah (2.22.0)
@@ -60,7 +61,7 @@ GEM
     method_source (1.0.0)
     mime-types (3.5.2)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2023.1205)
+    mime-types-data (3.2024.0206)
     mini_portile2 (2.8.5)
     mysql2 (0.5.6)
     nenv (0.3.0)
@@ -74,6 +75,7 @@ GEM
     parser (3.3.0.5)
       ast (~> 2.4.1)
       racc
+    paru (1.2.0)
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
@@ -98,8 +100,8 @@ GEM
     rspec-mocks (3.13.0)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-support (3.13.0)
-    rubocop (1.60.2)
+    rspec-support (3.13.1)
+    rubocop (1.61.0)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
@@ -110,22 +112,25 @@ GEM
       rubocop-ast (>= 1.30.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.30.0)
-      parser (>= 3.2.1.0)
+    rubocop-ast (1.31.1)
+      parser (>= 3.3.0.4)
     rubocop-capybara (2.20.0)
       rubocop (~> 1.41)
     rubocop-factory_bot (2.25.1)
       rubocop (~> 1.41)
+    rubocop-performance (1.18.0)
+      rubocop (>= 1.7.0, < 2.0)
+      rubocop-ast (>= 0.4.0)
     rubocop-rake (0.6.0)
       rubocop (~> 1.0)
-    rubocop-rspec (2.26.1)
+    rubocop-rspec (2.27.0)
       rubocop (~> 1.40)
       rubocop-capybara (~> 2.17)
       rubocop-factory_bot (~> 2.22)
     ruby-progressbar (1.13.0)
-    rugged (1.7.1)
+    rugged (1.7.2)
     shellany (0.0.1)
-    thor (1.3.0)
+    thor (1.3.1)
     twitter-text (1.14.7)
       unf (~> 0.1.0)
     unf (0.1.4)
@@ -145,6 +150,7 @@ DEPENDENCIES
   rake
   rspec
   rubocop
+  rubocop-performance
   rubocop-rake
   rubocop-rspec
 

README.md

@@ -2,60 +2,80 @@
 
 [![Maintainability](https://api.codeclimate.com/v1/badges/7fe3ef34e09ba8133424/maintainability)](https://codeclimate.com/github/Dahie/caramelize/maintainability)
 
-Caramelize is a compact and flexible wiki content migration tool. It is intended for easi transfer of content from legacy wikis. With caramelize you can create your own export configurations and migrate your data into a git-based [gollum](https://github.com/github/gollum) wiki retaining all your history and gaining the most flexible access to your wiki content.
+Caramelize is a compact and flexible wiki migration tool. It is intended for easy transfer of content from legacy wikis. With caramelize you can create your own export configurations and migrate your page revisions into a git repository of markdown files. This retains all your history and you gain the most flexible access to your wiki content available for use with git-based wikis like [gollum](https://github.com/github/gollum), [Otter Wiki](https://github.com/redimp/otterwiki), [Wiki.js](https://js.wiki/) or [Obsidian](https://obsidian.md/).
+
+By default, it ships with configurations for [WikkaWiki](http://wikkawiki.org/) and [Redmine](http://www.redmine.org/).
 
-In the future more target wikis may be added. For the moment migration is supported for [WikkaWiki](http://wikkawiki.org/) and [Redmine](http://www.redmine.org/)-Wiki.
 
 ## Usage
 
 ### Installation
 
-    $ gem install caramelize
+```sh
+$ gem install caramelize
+```
 
 Install the latest release of caramelize using RubyGems.
+Requires pandoc to be installed.
 
 ### Use
 
-    $ caramelize create
+```sh
+$ caramelize create
+```
 
 Creates a template configuration file "caramel.rb". This includes documentation on how to use the preset Wiki-connectors and how to write addition customized connectors. More about this below.
 
-    $ caramelize run
+```sh
+$ caramelize run
+```
 
 Will start the wiki migration based on the configuration file. These are either found in predefined paths (./caramel.rb, ./config.rb, …), or passed as argument, as below.
 
-    $ caramelize doctor
+```sh
+$ caramelize doctor
+```
 
 Can be used to assess the quality of your wiki conversion. It'll help you see
 how many wiki links may be broken and how many pages were orphaned.
 
-    $ caramelize help
+```sh
+$ caramelize help
+```
 
 Returns help information.
 
-	$ caramelize version
+```sh
+$ caramelize version
+```
 
 Returns version and release information.
 
 ### Options
 
-    $ caramelize create --config my_caramel_configuration.rb
+```sh
+$ caramelize create --config my_caramel_configuration.rb
+```
 
 Creates an example configuration by the given name.
 
-    $ caramelize run --config my_caramel_configuration.rb
+```sh
+$ caramelize run --config my_caramel_configuration.rb
+```
 
 Executes the given configuration.
 
-    $ caramelize --verbose [command]
+```sh
+$ caramelize --verbose [command]
+```
 
 Displays more verbose output to the command line.
 
 ## Content migration
 
 ### Wiki support
 
-Caramelize comes with direct support for [WikkaWiki](http://wikkawiki.org/) and [Redmine](http://www.redmine.org/)-Wiki.
+Caramelize comes with direct support for [MediaWiki](https://www.mediawiki.org), [WikkaWiki](http://wikkawiki.org/) and [Redmine](http://www.redmine.org/)-Wiki.
 More custom wikis can be supported by creating a suitable configuration file.
 
 The wiki is exported to markdown files in a git-repository. This can be directly used as source for [gollum](https://github.com/github/gollum) wiki, [Otter Wiki](https://github.com/redimp/otterwiki), or if you don't care about the history even [Obsidian](https://obsidian.md/). 
@@ -76,7 +96,7 @@ Since wiki software may have special features, that are not common among other w
 
 ### Configuration recipes
 
-The `caramel.rb` configuration contains the settings on how to import the data of the existing wiki and how to convert it into the format required by caramelize to export to gollum.
+The `lib/caramelize/caramel.rb` configuration contains the settings on how to import the data of the existing wiki and how to convert it into the format required by caramelize to export to gollum.
 You also find the predefined definitions for importing from WikkaWiki and Redmine and and example for a custom import.
 
 Custom import allows you to import data from wikis that are not natively supported by caramelize. Defining your own wiki import requires a bit of knowledge on Ruby and MySQL as you setup the access to your wiki database and need to define how the data is to be transformed. Depending on the database model of the wiki this can be one simple call for all revisions in the database, or it can get more complicated with multiple mysql-calls as the database becomes more complex.
@@ -123,41 +143,54 @@ end
 
 In the end the `wiki` instance needs the `titles` and `revisions` filled.
 
-Some wikis don't have all necessary metadata saved in the revision. In this case additional database queries are necessary. **The configuration recipe is pure ruby code, that is included on execution. This gives you alot of freedom in writing your configuration, but also a lot of power to break things for yourself. Be advised.**
+Some wikis don't have all necessary metadata saved in the revision. In this case additional database queries are necessary. **The configuration recipe is pure ruby code, that is included on execution. This gives you a lot of freedom in writing your configuration, but also a lot of power to break things. Be advised.**
 
 I'm happy to give support on your recipes and I'd also like to extend caramelize with more wiki modules, if you send in your configurations (minus database credentials of course).
 
 ### Building
 
 This is how you can build caramelize, in case you'd like to develop it further. To get startered you'll need Bundler.
 
-    $ gem install bundler
+```sh
+$ gem install bundler
+```
 
 Clone or fork this repository and start building.
 
-    $ git clone git@github.com:Dahie/caramelize.git
-    $ gem build caramelize.gemspec
+```sh
+$ git clone git@github.com:Dahie/caramelize.git
+$ gem build caramelize.gemspec
+```
 
 Now to build and package the gem do
 
-    $ rake build
+```sh
+$ rake build
+```
 
 or
 
-    $ rake install
+```sh
+$ rake install
+```
 
 to install the new gem right to your system.
 
+Tests run with
+
+```sh
+$ rspec
+```
+
 ## Contributing to caramelize
 
-* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
+* Check out the latest main to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
 * Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
 * Fork the project
 * Start a feature/bugfix branch
 * Commit and push until you are happy with your contribution
 * Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
 
-
 ## Copyright
 
 Copyright (c) 2011-2024 Daniel Senff. See LICENSE.md for further details.

caramelize.gemspec

@@ -22,6 +22,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency('commander')
   spec.add_dependency('gollum-lib')
   spec.add_dependency('mysql2')
+  spec.add_dependency('paru')
   spec.add_dependency('ruby-progressbar')
 
   spec.metadata['rubygems_mfa_required'] = 'true'

lib/caramelize.rb

@@ -2,18 +2,20 @@
 
 require 'caramelize/version'
 require 'caramelize/page'
-require 'caramelize/content_transferer'
-require 'caramelize/filter_processor'
 require 'caramelize/database_connector'
+require 'caramelize/filters/add_newline_to_page_end'
+require 'caramelize/filter_processor'
+require 'caramelize/input_wiki/wiki'
+require 'caramelize/input_wiki/media_wiki'
+require 'caramelize/input_wiki/redmine_wiki'
+require 'caramelize/input_wiki/wikka_wiki'
+require 'caramelize/content_transferer'
 require 'caramelize/health_check'
 require 'caramelize/health_checks/home_page_check'
 require 'caramelize/health_checks/orphaned_pages_check'
 require 'caramelize/health_checks/page'
 require 'caramelize/output_wiki/gollum'
 require 'caramelize/services/page_builder'
-require 'caramelize/input_wiki/wiki'
-require 'caramelize/input_wiki/redmine_wiki'
-require 'caramelize/input_wiki/wikkawiki'
 
 Encoding.default_external = Encoding::UTF_8
 Encoding.default_internal = Encoding::UTF_8