add: SemanticMediaWiki 6.0.1 as local extension

mediawiki/extensions/SemanticMediaWiki/.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: 
+patreon: # Replace with a single Patreon username
+open_collective: smw
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: ['https://www.semantic-mediawiki.org/wiki/Sponsorship']

mediawiki/extensions/SemanticMediaWiki/.github/ISSUE_TEMPLATE/add-tester.md

@@ -0,0 +1,10 @@
+---
+name: Membership
+about: Add a tester
+title: Please add me as a tester
+labels: ''
+assignees: ''
+
+---
+
+

mediawiki/extensions/SemanticMediaWiki/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,24 @@
+---
+name: Bug report
+about: Report a bug
+title: ''
+labels: bug
+assignees: ''
+
+---
+
+### Setup
+
+- SMW version:
+- MW version:
+- PHP version:
+- DB system (MySQL, Blazegraph, etc.) and version:
+
+### Issue
+
+Detailed description of the issue and a [stack trace](https://www.semantic-mediawiki.org/wiki/Help:Identifying_bugs) if applicable:
+
+```
+```
+
+Steps to reproduce the observation (recommendation is to use the [sandbox](https://sandbox.semantic-mediawiki.org)):

mediawiki/extensions/SemanticMediaWiki/.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,10 @@
+---
+name: Feature request
+about: Request a new feature
+title: ''
+labels: feature
+assignees: ''
+
+---
+
+

mediawiki/extensions/SemanticMediaWiki/.github/ISSUE_TEMPLATE/make-a-release.md

@@ -0,0 +1,33 @@
+---
+name: Make a release
+about: Keep track of the tasks for making a release
+title: Make the x.y.z realease
+labels: roadmap
+assignees: ''
+
+---
+
+## Tasks
+
+- [ ] Update release notes
+  - [ ] Add missing commits tagged with [releasenotes](https://github.com/SemanticMediaWiki/SemanticMediaWiki/pulls?q=is%3Apr+is%3Aopen+label%3Areleasenotes)
+  - [ ] Add RELEASE-NOTES to /docs/releasenotes/
+  - [ ] Update /docs/releasenotes/README
+- [ ] Update INSTALL - not just version number
+- [ ] Update composer.json
+- [ ] Update version number
+  - [ ] extension.json
+  - [ ] COMPATIBILITY
+- [ ] Create tag
+- [ ] Update `semantic-mediawiki.org`
+  - [ ] Run `composer update`
+  - [ ] Touch `LocalSettings.php`
+  - [ ] Update the SMW configuration
+  - [ ] Check [Special:Version](https://www.semantic-mediawiki.org/wiki/Special:Version)
+- [ ] Update [roadmap](https://www.semantic-mediawiki.org/wiki/Roadmap) if applicable
+- [ ] Update version on wikidata, en, mw, and Free Software Directory
+- [ ] Announce on wiki 
+- [ ] <s>Announce via mail (use series instead of branch as wording)</s>
+- [ ] <s>Do a release tweet on Twitter (Also mention if it is a feature, maintenance or security release)</s>
+- [ ] Close milestone
+- [ ] Update project

mediawiki/extensions/SemanticMediaWiki/.github/ISSUE_TEMPLATE/question.md

@@ -0,0 +1,10 @@
+---
+name: Question
+about: Ask a question
+title: ''
+labels: question
+assignees: ''
+
+---
+
+

mediawiki/extensions/SemanticMediaWiki/.github/dependabot.yml

@@ -0,0 +1,8 @@
+# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "composer"
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "daily"

mediawiki/extensions/SemanticMediaWiki/.github/workflows/main.yml

@@ -0,0 +1,83 @@
+name: build
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+  workflow_dispatch:
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    continue-on-error: ${{ matrix.experimental }}
+
+    strategy:
+      matrix:
+        include:
+          - mediawiki_version: '1.43'
+            php_version: 8.1
+            database_type: mysql
+            database_image: "mariadb:11.2"
+            coverage: true
+            experimental: false
+          - mediawiki_version: '1.43'
+            php_version: 8.2
+            database_type: mysql
+            database_image: "mariadb:11.2"
+            coverage: false
+            experimental: false
+          - mediawiki_version: '1.43'
+            php_version: 8.3
+            database_type: mysql
+            database_image: "mariadb:11.2"
+            coverage: false
+            experimental: false
+          - mediawiki_version: '1.44'
+            php_version: 8.3
+            database_type: mysql
+            database_image: "mariadb:11.2"
+            coverage: false
+            experimental: true
+          - mediawiki_version: '1.44'
+            php_version: 8.3
+            database_type: mysql
+            database_image: "mariadb:11.8"
+            coverage: false
+            experimental: true
+
+    env:
+      MW_VERSION: ${{ matrix.mediawiki_version }}
+      PHP_VERSION: ${{ matrix.php_version }}
+      DB_TYPE: ${{ matrix.database_type }}
+      DB_IMAGE: ${{ matrix.database_image }}
+      PHP_EXTENSIONS: pcntl
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+                   
+      - name: Update submodules
+        run: git submodule update --init --remote
+
+      - name: Run tests
+        run: make ci
+        if: matrix.coverage == false
+
+      - name: Run unit tests with coverage
+        run: make ci-coverage COMPOSER_PARAMS="-- --testsuite=semantic-mediawiki-unit"
+        if: matrix.coverage == true
+
+      - name: Run tests with coverage
+        run: make ci-coverage COMPOSER_PARAMS="-- --testsuite semantic-mediawiki-check,semantic-mediawiki-data-model,semantic-mediawiki-integration,semantic-mediawiki-import,semantic-mediawiki-structure"
+        if: matrix.coverage == true
+
+      - name: Upload code coverage
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: coverage/php/coverage.xml
+        if: matrix.coverage == true

mediawiki/extensions/SemanticMediaWiki/.github/workflows/tesa.yml

@@ -0,0 +1,44 @@
+name: Tesa
+
+on:
+  push:
+    paths:
+      - .github/workflows/tesa.yml
+      - Tesa/**
+      - composer.json
+
+jobs:
+  phpunit:
+    name: "PHPUnit PHP ${{ matrix.php }}"
+
+    strategy:
+      matrix:
+        php: [ 8.1, 8.2, 8.3, 8.4 ]
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup PHP
+        uses: shivammathur/setup-php@v2
+        with:
+          php-version: ${{ matrix.php }}
+          extensions: mbstring, intl
+          tools: composer, cs2pr
+          coverage: pcov
+
+      - name: Cache Composer cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.composer/cache
+          key: composer-cache-php${{ matrix.php }}
+
+      - name: Composer install
+        run: composer install --no-progress --no-interaction --prefer-dist --optimize-autoloader
+
+      - name: Add PHPUnit
+        run: composer require phpunit/phpunit:^9
+
+      - name: PHPUnit
+        run: vendor/bin/phpunit -c Tesa/phpunit.xml.dist

mediawiki/extensions/SemanticMediaWiki/.gitignore

@@ -0,0 +1,38 @@
+# -*- mode: gitignore; -*-
+!.*
+
+*~
+\#*\#
+*.kate-swp
+.\#*
+
+[._]*.sw[a-p]
+[._]s[a-rt-v][a-z]
+[._]ss[a-gi-z]
+[._]sw[a-p]
+
+vendor/
+extensions/
+build/logs/
+build/release/build-*
+
+composer.phar
+composer.lock
+
+.idea/
+.smw.json
+.phpunit.result.cache
+src/Tesa/.phpunit.result.cache
+
+composer
+installer
+.env
+.envrc
+conf
+expected
+
+# Operating systems
+## Mac OS X
+.DS_Store
+## Windows
+Thumbs.db

mediawiki/extensions/SemanticMediaWiki/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "build"]
+	path = build
+	url = https://github.com/gesinn-it-pub/docker-compose-ci.git

mediawiki/extensions/SemanticMediaWiki/.jshintignore

@@ -0,0 +1,8 @@
+# third-party libs
+res/jquery/
+
+# Those files need overall re-factoring and qunit tests
+res/smw/special/ext.smw.special.ask.js
+res/smw/special/ext.smw.special.browse.js
+res/smw/special/ext.smw.special.property.js
+res/smw/util/ext.smw.util.autocomplete.js

mediawiki/extensions/SemanticMediaWiki/.jshintrc

@@ -0,0 +1,32 @@
+{
+	"predef": [
+		"mediaWiki",
+		"semanticMediaWiki",
+		"jQuery",
+		"QUnit"
+	],
+
+	"bitwise": false, // prohibits the use of bitwise operators such as ^ (XOR), | (OR)
+	"camelcase": false, // allows you to force all variable names to use either camelCase style
+	"curly": true, // requires you to always put curly braces around blocks in loops and conditionals
+	"eqeqeq": true, //  prohibits the use of == and != in favor of === and !==
+	"forin": false, //  requires all for in loops to filter object's items
+	"immed": true, // prohibits the use of immediate function invocations without wrapping them
+	"latedef": true, // prohibits the use of a variable before it was defined
+	"newcap": false, // requires you to capitalize names of constructor functions
+	"noarg": true, // prohibits the use of arguments.caller and arguments.callee
+	"noempty": true, // warns when you have an empty block in your code
+	"nonew": false, // prohibits the use of constructor functions for side-effects
+	"quotmark": "single", // enforces the consistency of quotation marks used throughout your code
+	"regexp": false, // prohibits the use of unsafe . in regular expressions
+	"undef": false, // prohibits the use of explicitly undeclared variables
+	"unused": true, // warns when you define and never use your variables
+	"strict": true, // requires all functions to run in EcmaScript 5's strict mode
+	"trailing": true, // makes it an error to leave a trailing whitespace in your code
+
+	"laxbreak": true, // suppresses most of the warnings about possibly unsafe line breakings
+	"smarttabs": true, // suppresses warnings about mixed tabs
+	"multistr": true, // suppresses warnings about multi-line strings
+
+	"browser": true // defines globals exposed by modern browsers
+}

mediawiki/extensions/SemanticMediaWiki/.phpcs.xml

@@ -0,0 +1,74 @@
+<?xml version="1.0"?>
+<ruleset>
+	<rule ref="./vendor/mediawiki/mediawiki-codesniffer/MediaWiki">
+		<exclude name="Generic.Files.LineLength.TooLong" />
+		<exclude name="MediaWiki.Commenting.FunctionComment.MissingDocumentationPrivate" />
+		<exclude name="MediaWiki.Commenting.FunctionComment.MissingParamTag" />
+		<exclude name="MediaWiki.Commenting.FunctionComment.MissingReturn" />
+		<exclude name="MediaWiki.NamingConventions.LowerCamelFunctionsName.FunctionName" />
+		<exclude name="MediaWiki.Commenting.FunctionComment.NoParamType" />
+		<exclude name="MediaWiki.Files.ClassMatchesFilename.NotMatch" />
+		<exclude name="MediaWiki.Commenting.FunctionComment.MissingDocumentationPublic" />
+		<exclude name="MediaWiki.Commenting.FunctionComment.MissingDocumentationProtected" />
+		<exclude name="MediaWiki.Commenting.FunctionComment.MissingDocumentationPrivate" />
+		<exclude name="Generic.CodeAnalysis.AssignmentInCondition.FoundInWhileCondition" /> 
+		<exclude name="Squiz.Classes.ValidClassName.NotCamelCaps" />
+		<exclude name="Generic.CodeAnalysis.AssignmentInCondition.Found" />
+		<exclude name="MediaWiki.Commenting.PropertyDocumentation.MissingDocumentationPrivate" />
+		<exclude name="MediaWiki.Commenting.PropertyDocumentation.MissingDocumentationProtected" />
+		<exclude name="MediaWiki.Commenting.PropertyDocumentation.MissingDocumentationPublic" />
+		<exclude name="MediaWiki.PHPUnit.AssertEmpty.AssertEmptyUsed" />
+		<exclude name="MediaWiki.Commenting.PropertyDocumentation.WrongStyle" />
+		<exclude name="Generic.Files.OneObjectStructurePerFile.MultipleFound" />
+		<exclude name="MediaWiki.Commenting.LicenseComment.InvalidLicenseTag" />
+		<exclude name="MediaWiki.PHPUnit.DeprecatedPHPUnitMethods.AssertInternalTypeGeneric" />
+		<exclude name="MediaWiki.Commenting.PropertyDocumentation.MissingVar" />
+		<exclude name="MediaWiki.Commenting.MissingCovers.MissingCovers" />
+		<exclude name="PSR12.Properties.ConstantVisibility.NotFound" />
+		<exclude name="MediaWiki.WhiteSpace.SpaceBeforeSingleLineComment.NewLineComment" />
+		<exclude name="MediaWiki.PHPUnit.SetMethods.SetMethods" />
+		<exclude name="MediaWiki.Commenting.FunctionComment.WrongStyle" />
+		<exclude name="MediaWiki.VariableAnalysis.MisleadingGlobalNames.Misleading$wgCapitalLinks" />
+		<exclude name="MediaWiki.NamingConventions.ValidGlobalName.allowedPrefix" />
+		<exclude name="MediaWiki.Usage.AssignmentInReturn.AssignmentInReturn" />
+		<exclude name="MediaWiki.Commenting.PhpunitAnnotations.ForbiddenTest" />
+		<exclude name="MediaWiki.Commenting.FunctionAnnotations.UnrecognizedAnnotation" />
+		<exclude name="MediaWiki.Commenting.PhpunitAnnotations.NotTestClass" />
+		<exclude name="MediaWiki.Commenting.PhpunitAnnotations.NotTestFunction" />
+		<exclude name="MediaWiki.Commenting.FunctionComment.MissingParamName" />
+		<exclude name="MediaWiki.Commenting.PhpunitAnnotations.NotClass" />
+		<exclude name="MediaWiki.Commenting.PhpunitAnnotations.ForbiddenTestdox" />
+		<exclude name="MediaWiki.Usage.ForbiddenFunctions.is_resource" />
+		<exclude name="MediaWiki.Usage.ForbiddenFunctions.proc_open" />
+		<exclude name="MediaWiki.Usage.ForbiddenFunctions.exec" />
+		<exclude name="MediaWiki.Files.ClassMatchesFilename.WrongCase" />
+		<exclude name="MediaWiki.Usage.SuperGlobalsUsage.SuperGlobals" />
+		<exclude name="MediaWiki.Commenting.FunctionComment.ExtraParamComment" />
+		<exclude name="Generic.PHP.NoSilencedErrors.Discouraged" />
+		<exclude name="MediaWiki.Usage.DeprecatedGlobalVariables.Deprecated$wgTitle" />
+		<exclude name="MediaWiki.Commenting.PropertyDocumentation.MissingVarType" />
+		<exclude name="MediaWiki.Commenting.VariadicArgument.SuperfluousVariadicArgComment" />
+		<exclude name="MediaWiki.Usage.ForbiddenFunctions.assert" />
+		<exclude name="Squiz.Scope.MethodScope.Missing" />
+		<exclude name="Generic.NamingConventions.UpperCaseConstantName.ClassConstantNotUpperCase" />
+		<exclude name="PSR2.Methods.MethodDeclaration.Underscore" />
+		<exclude name="MediaWiki.Commenting.FunctionComment.MissingReturnType" />
+		<exclude name="MediaWiki.Usage.ExtendClassUsage.FunctionConfigUsage" />
+		<exclude name="MediaWiki.NamingConventions.PrefixedGlobalFunctions.allowedPrefix" />
+		<exclude name="MediaWiki.Commenting.LicenseComment.LicenseTagNonFileComment" />
+		<exclude name="MediaWiki.VariableAnalysis.MisleadingGlobalNames.Misleading$wgCategoryCollation" />
+	</rule>
+	<rule ref="MediaWiki.NamingConventions.ValidGlobalName">
+		<properties>
+			<property name="allowedPrefixes" type="array" value="eg,wg" />
+		</properties>
+	</rule>
+	<rule ref="MediaWiki.Commenting.FunctionComment.ObjectTypeHintReturn">
+		<severity>0</severity>
+	</rule>
+	<file>.</file>
+	<exclude-pattern>/(vendor|conf)/</exclude-pattern>
+	<exclude-pattern type="relative">extensions/*</exclude-pattern>
+	<arg name="extensions" value="php"/>
+	<arg name="encoding" value="UTF-8"/>
+</ruleset>

mediawiki/extensions/SemanticMediaWiki/COPYING

@@ -0,0 +1,355 @@
+Semantic MediaWiki (SMW) is a free, open-source extension to MediaWiki
+that lets you store and query data within the wiki's pages.
+
+Copyright (C) 2005
+
+
+The license text below "====" applies to all files within this distribution,
+other than those that are in a directory which contains files named "LICENSE"
+or "COPYING", or a subdirectory thereof. For those files, the license text
+contained in said file overrides any license information contained in directories
+of smaller depth. Alternative licenses are typically used for software that
+is provided by external parties, and merely packaged with the Semantic MediaWiki
+release for convenience.
+
+====
+
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+                            NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.

mediawiki/extensions/SemanticMediaWiki/DefaultSettings.php

@@ -0,0 +1,13 @@
+<?php
+
+/**
+ * This file is a back-compat shim and is not used by SMW.
+ *
+ * If you would like to see the DefaultSettings.php that was here before, look in
+ * includes/DefaultSettings.php.
+ *
+ * @deprecated 4.0.0
+ * @codeCoverageIgnore
+ */
+require_once __DIR__ . "/includes/SemanticMediaWiki.php";
+return SemanticMediaWiki::getDefaultSettings();

mediawiki/extensions/SemanticMediaWiki/Makefile

@@ -0,0 +1,29 @@
+-include .env
+export
+
+# setup for docker-compose-ci build directory
+# delete "build" directory to update docker-compose-ci
+
+ifeq (,$(wildcard ./build/))
+    $(shell git submodule update --init --remote)
+endif
+
+EXTENSION=SemanticMediaWiki
+
+# docker images
+MW_VERSION?=1.43
+PHP_VERSION?=8.1
+DB_TYPE?=mysql
+DB_IMAGE?="mariadb:11.2"
+PHP_EXTENSIONS?=pcntl
+
+# composer
+# Enables "composer update" inside of extension
+COMPOSER_EXT?=true
+
+# nodejs
+# Enables node.js related tests and "npm install"
+# NODE_JS?=true
+
+# check for build dir and git submodule init if it does not exist
+include build/Makefile

mediawiki/extensions/SemanticMediaWiki/README.md

@@ -0,0 +1,122 @@
+# Semantic MediaWiki
+
+[![CI](https://github.com/SemanticMediaWiki/SemanticMediaWiki/actions/workflows/main.yml/badge.svg)](https://github.com/SemanticMediaWiki/SemanticMediaWiki/actions/workflows/main.yml)
+[![Latest Stable Version](https://poser.pugx.org/mediawiki/semantic-media-wiki/v/stable)](https://packagist.org/packages/mediawiki/semantic-media-wiki)
+[![Download count](https://poser.pugx.org/mediawiki/semantic-media-wiki/downloads)](https://packagist.org/packages/mediawiki/semantic-media-wiki)
+[![codecov](https://codecov.io/gh/SemanticMediaWiki/SemanticMediaWiki/graph/badge.svg?token=yl1GVLwRwo)](https://codecov.io/gh/SemanticMediaWiki/SemanticMediaWiki)
+[![License](https://poser.pugx.org/mediawiki/semantic-media-wiki/license)](COPYING)
+
+**Semantic MediaWiki** (a.k.a. SMW) is a free, open-source extension to [MediaWiki](https://www.semantic-mediawiki.org/wiki/MediaWiki) – the wiki software that
+powers Wikipedia – that lets you store and query data within the wiki's pages.
+
+Semantic MediaWiki is also a full-fledged framework with
+many spinoff extensions that can turn a wiki into a powerful and flexible
+knowledge management system. All data created within SMW can easily be
+published via the [Semantic Web](https://www.semantic-mediawiki.org/wiki/Semantic_Web),
+allowing other systems to use this data seamlessly.
+
+For a better understanding of how Semantic MediaWiki works, have a look at [deployed in 5 min](https://vimeo.com/82255034)
+and the [Sesame](https://vimeo.com/126392433), [Fuseki ](https://vimeo.com/118614078) triplestore video, or
+browse the [wiki](https://www.semantic-mediawiki.org) for a more comprehensive introduction.
+
+## Requirements
+
+Semantic MediaWiki requires MediaWiki and its dependencies, such as PHP. For supported versions,
+see the [compatibility matrix](docs/COMPATIBILITY.md).
+
+## Installation
+
+The recommended way to install Semantic MediaWiki is by using [Composer][composer]. See the detailed
+[installation guide](docs/INSTALL.md) as well as the information on [compatibility](docs/COMPATIBILITY.md).
+
+## Documentation
+
+Most of the documentation can be found on the [Semantic MediaWiki wiki](https://www.semantic-mediawiki.org).
+A small core of documentation also comes bundled with the software itself. This documentation is minimalistic
+and less explanatory than what can be found on the SMW wiki. However, It is always kept up to date and applies
+to the version of the code it bundles with. The most critical files are linked below.
+
+* [User documentation](docs/README.md)
+* [Technical documentation](docs/technical/README.md)
+* [Hacking Semantic MediaWiki](docs/architecture/README.md)
+
+## Support
+
+[![Chatroom](https://www.semantic-mediawiki.org/w/thumb.php?f=Comment-alt-solid.svg&width=35)](https://www.semantic-mediawiki.org/wiki/Semantic_MediaWiki_chatroom)
+[![Twitter](https://www.semantic-mediawiki.org/w/thumb.php?f=Twitter-square.svg&width=35)](https://twitter.com/#!/semanticmw)
+[![Facebook](https://www.semantic-mediawiki.org/w/thumb.php?f=Facebook-square.svg&width=35)](https://www.facebook.com/pages/Semantic-MediaWiki/160459700707245)
+[![LinkedIn](https://www.semantic-mediawiki.org/w/thumb.php?f=LinkedIn-square.svg&width=35)]([https://twitter.com/#!/semanticmw](https://www.linkedin.com/groups/2482811/))
+[![YouTube](https://www.semantic-mediawiki.org/w/thumb.php?f=Youtube-square.svg&width=35)](https://www.youtube.com/c/semanticmediawiki)
+[![Mailing lists](https://www.semantic-mediawiki.org/w/thumb.php?f=Envelope-square.svg&width=35)](https://www.semantic-mediawiki.org/wiki/Semantic_MediaWiki_mailing_lists)
+
+Primary support channels:
+
+* [User mailing list](https://sourceforge.net/projects/semediawiki/lists/semediawiki-user) - for user questions
+* [SMW chat room](https://www.semantic-mediawiki.org/wiki/Semantic_MediaWiki_chatroom) - for questions and developer discussions
+* [Issue tracker](https://github.com/SemanticMediaWiki/SemanticMediaWiki/issues) - for bug reports
+
+## Contributing
+
+Many people have contributed to SMW. A list of people who have made contributions in the past can
+be found [here][contributors] or on the [wiki for Semantic MediaWiki](https://www.semantic-mediawiki.org/wiki/Help:SMW_Project#Contributors).
+The overview on [how to contribute](https://github.com/SemanticMediaWiki/SemanticMediaWiki/blob/master/docs/CONTRIBUTING.md)
+provides information on the different ways available to do so.
+
+If you want to contribute work to the project, please subscribe to the developer's mailing list and
+have a look at the contribution guidelines.
+
+* [File an issue](https://github.com/SemanticMediaWiki/SemanticMediaWiki/issues)
+* [Submit a pull request](https://github.com/SemanticMediaWiki/SemanticMediaWiki/pulls)
+* Ask a question on [the mailing list](https://www.semantic-mediawiki.org/wiki/Mailing_list)
+
+## Tests
+
+This extension is tested using [GitHub Actions for Continuous Integration (CI)](https://github.com/SemanticMediaWiki/SemanticMediaWiki/actions). Each time changes are pushed to the repository, GitHub Actions automatically runs a series of tests to ensure the code remains reliable and functional.
+
+> **INFO**:
+> This repository contains submodules. Make sure to clone with `--recursive` option in Git.
+>
+> ```
+> git clone --recursive <REPO>
+> ```
+> 
+> If not done when cloning, it can be done by
+>
+> ```
+> git submodule init
+> git submodule update
+> ```
+
+### Step 1: Clone the Repository
+
+### Step 2: Ensure test container is running
+This repository supports ["docker-compose-ci" based CI and testing for MediaWiki extensions](https://github.com/gesinn-it-pub/docker-compose-ci).
+
+The "docker-compose-ci" repository has already been integrated into the Semantic MediaWiki repository as a Git submodule. It uses "Make" as main entry point and command line interface.
+
+Ensure, you have `Make` and `Docker` installed:
+```
+make --version
+docker --version
+```
+
+### Step 3: Run lint, phpcs and tests
+
+```
+make ci
+```
+
+For more information about
+- docker-compose-ci, see https://github.com/gesinn-it-pub/docker-compose-ci
+- tests in Semantic MediaWiki in general, see the [test documentation](/tests/README.md#running-tests).
+
+## License
+
+[GNU General Public License, version 2 or later][gpl-licence]. The COPYING file explains SMW's copyright and license.
+
+[contributors]: https://github.com/SemanticMediaWiki/SemanticMediaWiki/graphs/contributors
+[travis]: https://travis-ci.org/SemanticMediaWiki/SemanticMediaWiki
+[mw-testing]: https://www.mediawiki.org/wiki/Manual:PHP_unit_testing
+[gpl-licence]: https://www.gnu.org/copyleft/gpl.html
+[composer]: https://getcomposer.org/
+[smw-installation]: https://www.semantic-mediawiki.org/wiki/Help:Installation

mediawiki/extensions/SemanticMediaWiki/Tesa/LICENSE

@@ -0,0 +1,340 @@
+GNU GENERAL PUBLIC LICENSE
+                       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc., <http://fsf.org/>
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+                            NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    {description}
+    Copyright (C) {year}  {fullname}
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  {signature of Ty Coon}, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.
+

mediawiki/extensions/SemanticMediaWiki/Tesa/README.md

@@ -0,0 +1,118 @@
+# Tesa (text sanitizer)
+
+[![Build Status](https://secure.travis-ci.org/onoi/tesa.svg?branch=master)](http://travis-ci.org/onoi/tesa)
+[![Code Coverage](https://scrutinizer-ci.com/g/onoi/tesa/badges/coverage.png?b=master)](https://scrutinizer-ci.com/g/onoi/tesa/?branch=master)
+[![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/onoi/tesa/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/onoi/tesa/?branch=master)
+[![Latest Stable Version](https://poser.pugx.org/onoi/tesa/version.png)](https://packagist.org/packages/onoi/tesa)
+[![Packagist download count](https://poser.pugx.org/onoi/tesa/d/total.png)](https://packagist.org/packages/onoi/tesa)
+[![Dependency Status](https://www.versioneye.com/php/onoi:tesa/badge.png)](https://www.versioneye.com/php/onoi:tesa)
+
+The library contains a small collection of helper classes to support sanitization
+of text or string elements of arbitrary length with the aim to improve
+search match confidence during a query execution that is required by [Semantic MediaWiki][smw]
+project and is deployed independently.
+
+## Requirements
+
+- PHP 7.4
+- Recommended to enable the [ICU][icu] extension
+
+## Installation
+
+The recommended installation method for this library is by adding
+the following dependency to your [composer.json][composer].
+
+```json
+{
+	"require": {
+		"onoi/tesa": "~0.1"
+	}
+}
+```
+
+## Usage
+
+```php
+use Onoi\Tesa\SanitizerFactory;
+use Onoi\Tesa\Transliterator;
+use Onoi\Tesa\Sanitizer;
+
+$sanitizerFactory = new SanitizerFactory();
+
+$sanitizer = $sanitizerFactory->newSanitizer( 'A string that contains ...' );
+
+$sanitizer->reduceLengthTo( 200 );
+$sanitizer->toLowercase();
+
+$sanitizer->replace(
+	array( "'", "http://", "https://", "mailto:", "tel:" ),
+	array( '' )
+);
+
+$sanitizer->setOption( Sanitizer::MIN_LENGTH, 4 );
+$sanitizer->setOption( Sanitizer::WHITELIST, array( 'that' ) );
+
+$sanitizer->applyTransliteration(
+	Transliterator::DIACRITICS | Transliterator::GREEK
+);
+
+$text = $sanitizer->sanitizeWith(
+	$sanitizerFactory->newGenericTokenizer(),
+	$sanitizerFactory->newNullStopwordAnalyzer(),
+	$sanitizerFactory->newNullSynonymizer()
+);
+
+```
+
+- `SanitizerFactory` is expected to be the sole entry point for services and instances
+  when used outside of this library
+- `IcuWordBoundaryTokenizer` is a preferred tokenizer in case the [ICU][icu] extension is available
+- `NGramTokenizer` is provided to increase CJK match confidence in case the
+  back-end does not provide an explicit ngram tokenizer
+- `StopwordAnalyzer` together with a `LanguageDetector` is provided as a means to
+  reduce ambiguity of frequent "noise" words from a possible search index
+- `Synonymizer` currently only provides an interface
+
+## Contribution and support
+
+If you want to contribute work to the project please subscribe to the
+developers mailing list and have a look at the [contribution guidelinee](/CONTRIBUTING.md). A list
+of people who have made contributions in the past can be found [here][contributors].
+
+* [File an issue](https://github.com/onoi/tesa/issues)
+* [Submit a pull request](https://github.com/onoi/tesa/pulls)
+
+## Tests
+
+The library provides unit tests that covers the core-functionality normally run by the
+[continues integration platform][travis]. Tests can also be executed manually using the
+`composer phpunit` command from the root directory.
+
+## Release notes
+
+- 0.1.0 Initial release (2016-08-07)
+ - Added `SanitizerFactory` with support for a
+ - `Tokenizer`, `LanguageDetector`, `Synonymizer`, and `StopwordAnalyzer` interface
+
+## Acknowledgments
+
+- The `Transliterator` uses the same diacritics conversion table as http://jsperf.com/latinize
+  (except the German diaeresis ä, ü, and ö)
+- The stopwords used by the `StopwordAnalyzer` have been collected from different sources, each `json`
+  file identifies its origin
+- `CdbStopwordAnalyzer` relies on `wikimedia/cdb` to avoid using an external database or cache
+  layer (with extra stopwords being available [here](https://github.com/6/stopwords-json))
+- `JaTinySegmenterTokenizer` is based on the work of Taku Kudo and his [tiny_segmenter.js](http://chasen.org/~taku/software/TinySegmenter)
+- `TextCatLanguageDetector` uses the [`wikimedia/textcat`][textcat] library to make predictions about a language
+
+## License
+
+[GNU General Public License 2.0 or later][license].
+
+[composer]: https://getcomposer.org/
+[contributors]: https://github.com/onoi/tesa/graphs/contributors
+[license]: https://www.gnu.org/copyleft/gpl.html
+[travis]: https://travis-ci.org/onoi/tesa
+[smw]: https://github.com/SemanticMediaWiki/SemanticMediaWiki/
+[icu]: http://php.net/manual/en/intro.intl.php
+[textcat]: https://github.com/wikimedia/wikimedia-textcat

mediawiki/extensions/SemanticMediaWiki/Tesa/phpunit.xml.dist

@@ -0,0 +1,27 @@
+<phpunit backupGlobals="false"
+         backupStaticAttributes="false"
+         bootstrap="tests/bootstrap.php"
+         cacheTokens="false"
+         colors="true"
+         convertErrorsToExceptions="true"
+         convertNoticesToExceptions="true"
+         convertWarningsToExceptions="true"
+         stopOnError="false"
+         stopOnFailure="false"
+         stopOnIncomplete="false"
+         stopOnSkipped="false"
+         verbose="true">
+    <testsuites>
+        <testsuite name="tesa-unit">
+            <directory>tests/phpunit/Unit</directory>
+        </testsuite>
+        <testsuite name="tesa-integration">
+            <directory>tests/phpunit/Integration</directory>
+        </testsuite>
+    </testsuites>
+    <filter>
+        <whitelist addUncoveredFilesFromWhitelist="true">
+            <directory suffix=".php">src</directory>
+        </whitelist>
+    </filter>
+</phpunit>

mediawiki/extensions/SemanticMediaWiki/Tesa/src/CharacterExaminer.php

@@ -0,0 +1,61 @@
+<?php
+
+namespace Onoi\Tesa;
+
+/**
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class CharacterExaminer {
+
+	const CYRILLIC = 'CYRILLIC';
+	const LATIN = 'LATIN';
+	const HIRAGANA_KATAKANA = 'HIRAGANA_KATAKANA';
+	const HANGUL = 'HANGUL';
+	const CJK_UNIFIED = 'CJK_UNIFIED';
+	const HAN = 'HAN';
+
+	/**
+	 * @see http://jrgraphix.net/research/unicode_blocks.php
+	 * @since 0.1
+	 *
+	 * @param string $type
+	 * @param string $text
+	 *
+	 * @return bool
+	 */
+	public static function contains( $type, $text ) {
+		if ( $type === self::CYRILLIC ) {
+			return preg_match( '/\p{Cyrillic}/u', $text ) > 0;
+		}
+
+		if ( $type === self::LATIN ) {
+			return preg_match( '/\p{Latin}/u', $text ) > 0;
+		}
+
+		if ( $type === self::HAN ) {
+			return preg_match( '/\p{Han}/u', $text ) > 0;
+		}
+
+		if ( $type === self::HIRAGANA_KATAKANA ) {
+			return preg_match( '/[\x{3040}-\x{309F}]/u', $text ) > 0 || preg_match( '/[\x{30A0}-\x{30FF}]/u', $text ) > 0; // isHiragana || isKatakana
+		}
+
+		if ( $type === self::HANGUL ) {
+			return preg_match( '/[\x{3130}-\x{318F}]/u', $text ) > 0 || preg_match( '/[\x{AC00}-\x{D7AF}]/u', $text ) > 0;
+		}
+
+		// @see https://en.wikipedia.org/wiki/CJK_Unified_Ideographs
+		// Chinese, Japanese and Korean (CJK) scripts share common characters
+		// known as CJK characters
+
+		if ( $type === self::CJK_UNIFIED ) {
+			return preg_match( '/[\x{4e00}-\x{9fa5}]/u', $text ) > 0;
+		}
+
+		return false;
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/src/LanguageDetector/LanguageDetector.php

@@ -0,0 +1,22 @@
+<?php
+
+namespace Onoi\Tesa\LanguageDetector;
+
+/**
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+interface LanguageDetector {
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param string $text
+	 *
+	 * @return string|null
+	 */
+	public function detect( $text );
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/src/LanguageDetector/NullLanguageDetector.php

@@ -0,0 +1,24 @@
+<?php
+
+namespace Onoi\Tesa\LanguageDetector;
+
+/**
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class NullLanguageDetector implements LanguageDetector {
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param string $text
+	 *
+	 * @return null
+	 */
+	public function detect( $text ) {
+		return null;
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/src/LanguageDetector/TextCatLanguageDetector.php

@@ -0,0 +1,64 @@
+<?php
+
+namespace Onoi\Tesa\LanguageDetector;
+
+use TextCat;
+
+/**
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class TextCatLanguageDetector implements LanguageDetector {
+
+	const VERION = '0.1';
+
+	/**
+	 * @var TextCat
+	 */
+	private $textCat;
+
+	/**
+	 * @var array|null
+	 */
+	private $languageCandidates = null;
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param TextCat|null $textCat
+	 */
+	public function __construct( ?TextCat $textCat = null ) {
+		$this->textCat = $textCat;
+
+		if ( $this->textCat === null ) {
+			$this->textCat = new TextCat();
+		}
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param array $languageCandidates
+	 */
+	public function setLanguageCandidates( array $languageCandidates ) {
+		$this->languageCandidates = $languageCandidates;
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param string $text
+	 *
+	 * @return string|null
+	 */
+	public function detect( $text ) {
+		$languages = $this->textCat->classify( $text, $this->languageCandidates );
+		reset( $languages );
+
+		// For now, only return the best match
+		return key( $languages );
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/src/Normalizer.php

@@ -0,0 +1,97 @@
+<?php
+
+namespace Onoi\Tesa;
+
+/**
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class Normalizer {
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param string $text
+	 *
+	 * @param int $flag
+	 */
+	public static function applyTransliteration( $text, $flag = Transliterator::DIACRITICS ) {
+		return Transliterator::transliterate( $text, $flag );
+	}
+
+	/**
+	 * @see Localizer::convertDoubleWidth
+	 *
+	 * @since 0.1
+	 *
+	 * @param string $text
+	 *
+	 * @return string
+	 */
+	public static function convertDoubleWidth( $text ) {
+		static $full = null;
+		static $half = null;
+
+		// ,。／？《》〈〉；：“”＂〃＇｀［］｛｝＼｜～！－＝＿＋）（()＊…—─％￥＃
+		//,./?«»();:“”
+
+		if ( $full === null ) {
+			$fullWidth = "０１２３４５６７８９ＡＢＣＤＥＦＧＨＩＪＫＬＭＮＯＰＱＲＳＴＵＶＷＸＹＺａｂｃｄｅｆｇｈｉｊｋｌｍｎｏｐｑｒｓｔｕｖｗｘｙｚ";
+			$halfWidth = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
+
+			// http://php.net/manual/en/function.str-split.php, mb_str_split
+			$length = mb_strlen( $fullWidth, 'UTF-8' );
+			$full = [];
+
+			for ( $i = 0; $i < $length; $i += 1 ) {
+				$full[] = mb_substr( $fullWidth, $i, 1, 'UTF-8' );
+			}
+
+			$half = str_split( $halfWidth );
+		}
+
+		return str_replace( $full, $half, trim( $text ) );
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param string $text
+	 *
+	 * @return string
+	 */
+	public static function toLowercase( $text ) {
+		$encoding = mb_detect_encoding( $text ) ?: 'UTF-8';
+		return mb_strtolower( $text, $encoding );
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param string $text
+	 * @param int|null $length
+	 *
+	 * @return string
+	 */
+	public static function reduceLengthTo( $text, $length = null ) {
+		if ( $length === null || mb_strlen( $text ) <= $length ) {
+			return $text;
+		}
+
+		$encoding = mb_detect_encoding( $text ) ?: 'UTF-8';
+		$lastWholeWordPosition = $length;
+
+		if ( strpos( $text, ' ' ) !== false ) {
+			$lastWholeWordPosition = strrpos( mb_substr( $text, 0, $length, $encoding ), ' ' ); // last whole word
+		}
+
+		if ( $lastWholeWordPosition > 0 ) {
+			$length = $lastWholeWordPosition;
+		}
+
+		return mb_substr( $text, 0, $length, $encoding );
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/src/Sanitizer.php

@@ -0,0 +1,175 @@
+<?php
+
+namespace Onoi\Tesa;
+
+use Onoi\Tesa\StopwordAnalyzer\StopwordAnalyzer;
+use Onoi\Tesa\Synonymizer\Synonymizer;
+use Onoi\Tesa\Tokenizer\Tokenizer;
+
+/**
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class Sanitizer {
+
+	const WHITELIST = 'WHITELIST';
+	const MIN_LENGTH = 'MIN_LENGTH';
+
+	/**
+	 * Any change to the content of its data files should be reflected in a
+	 * version change (the version number does not necessarily correlate with
+	 * the library version)
+	 */
+	const VERSION = '0.2';
+
+	/**
+	 * @var string
+	 */
+	private $string = '';
+
+	/**
+	 * @var array
+	 */
+	private $whiteList = [];
+
+	/**
+	 * @var array
+	 */
+	private $minLength = 3;
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param string $string
+	 */
+	public function __construct( $string = '' ) {
+		$this->setText( $string );
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param string $name
+	 * @param mixed $value
+	 */
+	public function setOption( $name, $value ) {
+		if ( $name === self::WHITELIST && is_array( $value ) && $value !== [] ) {
+			$this->whiteList = array_fill_keys( $value, true );
+		}
+
+		if ( $name === self::MIN_LENGTH ) {
+			$this->minLength = (int)$value;
+		}
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param string $string
+	 */
+	public function setText( $string ) {
+		$this->string = $string;
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param int $flag
+	 */
+	public function applyTransliteration( $flag = Transliterator::DIACRITICS ) {
+		$this->string = Normalizer::applyTransliteration( $this->string, $flag );
+	}
+
+	/**
+	 * @see Localizer::convertDoubleWidth
+	 *
+	 * @since 0.1
+	 *
+	 * @param int $flag
+	 */
+	public function convertDoubleWidth() {
+		$this->string = Normalizer::convertDoubleWidth( $this->string );
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param Tokenizer $tokenizer
+	 * @param StopwordAnalyzer $stopwordAnalyzer
+	 *
+	 * @return string
+	 */
+	public function sanitizeWith( Tokenizer $tokenizer, StopwordAnalyzer $stopwordAnalyzer, Synonymizer $synonymizer ) {
+		// Treat non-words tokenizers (Ja,Zh*) differently
+		$minLength = $tokenizer->isWordTokenizer() ? $this->minLength : 1;
+
+		$words = $tokenizer->tokenize( $this->string );
+
+		if ( !$words || !is_array( $words ) ) {
+			return $this->string;
+		}
+
+		$index = [];
+		$pos = 0;
+
+		foreach ( $words as $key => $word ) {
+
+			$word = $synonymizer->synonymize( $word );
+
+			// If it is not an exemption and less than the required minimum length
+			// or identified as stop word it is removed
+			if ( !isset( $this->whiteList[$word] ) && ( mb_strlen( $word ) < $minLength || $stopwordAnalyzer->isStopWord( $word ) ) ) {
+				continue;
+			}
+
+			// Simple proximity, check for same words appearing next to each other
+			if ( isset( $index[$pos - 1] ) && $index[$pos - 1] === $word ) {
+				continue;
+			}
+
+			$index[] = trim( $word );
+			$pos++;
+		}
+
+		return implode( ' ', $index );
+	}
+
+	/**
+	 * @since 0.1
+	 */
+	public function toLowercase() {
+		$this->string = Normalizer::toLowercase( $this->string );
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param int $length
+	 */
+	public function reduceLengthTo( $length ) {
+		$this->string = Normalizer::reduceLengthTo( $this->string, $length );
+	}
+
+	/**
+	 * @see http://www.phpwact.org/php/i18n/utf-8#str_replace
+	 * @since 0.1
+	 *
+	 * @param string $search
+	 * @param string $replace
+	 */
+	public function replace( $search, $replace ) {
+		$this->string = str_replace( $search, $replace, $this->string );
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @return string
+	 */
+	public function __toString() {
+		return $this->string;
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/src/SanitizerFactory.php

@@ -0,0 +1,253 @@
+<?php
+
+namespace Onoi\Tesa;
+
+use Onoi\Tesa\LanguageDetector\NullLanguageDetector;
+use Onoi\Tesa\LanguageDetector\TextCatLanguageDetector;
+use Onoi\Tesa\StopwordAnalyzer\ArrayStopwordAnalyzer;
+use Onoi\Tesa\StopwordAnalyzer\CdbStopwordAnalyzer;
+use Onoi\Tesa\StopwordAnalyzer\NullStopwordAnalyzer;
+use Onoi\Tesa\StopwordAnalyzer\StopwordAnalyzer;
+use Onoi\Tesa\Synonymizer\NullSynonymizer;
+use Onoi\Tesa\Synonymizer\Synonymizer;
+use Onoi\Tesa\Tokenizer\CJKSimpleCharacterRegExTokenizer;
+use Onoi\Tesa\Tokenizer\GenericRegExTokenizer;
+use Onoi\Tesa\Tokenizer\IcuWordBoundaryTokenizer;
+use Onoi\Tesa\Tokenizer\JaCompoundGroupTokenizer;
+use Onoi\Tesa\Tokenizer\JaTinySegmenterTokenizer;
+use Onoi\Tesa\Tokenizer\NGramTokenizer;
+use Onoi\Tesa\Tokenizer\PunctuationRegExTokenizer;
+use Onoi\Tesa\Tokenizer\Tokenizer;
+
+/**
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class SanitizerFactory {
+
+	/**
+	 * @since 0.1
+	 *
+	 * @return Sanitizer
+	 */
+	public function newSanitizer( $text = '' ) {
+		return new Sanitizer( $text );
+	}
+
+	/* StopwordAnalyzer */
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param string|null $languageCode
+	 *
+	 * @return StopwordAnalyzer
+	 */
+	public function newStopwordAnalyzerByLanguage( $languageCode = null ) {
+		if ( $languageCode === null ) {
+			return $this->newNullStopwordAnalyzer();
+		}
+
+		$cdbStopwordAnalyzer = $this->newCdbStopwordAnalyzer(
+			$languageCode
+		);
+
+		return $cdbStopwordAnalyzer->isAvailable() ? $cdbStopwordAnalyzer : $this->newNullStopwordAnalyzer();
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @return StopwordAnalyzer
+	 */
+	public function newCdbStopwordAnalyzer( $languageCode = null ) {
+		return new CdbStopwordAnalyzer( CdbStopwordAnalyzer::getTargetByLanguage( $languageCode ) );
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param array $stopwords
+	 *
+	 * @return StopwordAnalyzer
+	 */
+	public function newArrayStopwordAnalyzer( array $stopwords = [] ) {
+		return new ArrayStopwordAnalyzer( $stopwords );
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @return StopwordAnalyzer
+	 */
+	public function newNullStopwordAnalyzer() {
+		return new NullStopwordAnalyzer();
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param string|null $languageCode
+	 *
+	 * @return Synonymizer
+	 */
+	public function newSynonymizerByLanguage( $languageCode = null ) {
+		if ( $languageCode === null ) {
+			return $this->newNullSynonymizer();
+		}
+
+		return $this->newNullSynonymizer();
+	}
+
+	/* Synonymizer */
+
+	/**
+	 * @since 0.1
+	 *
+	 * @return Synonymizer
+	 */
+	public function newNullSynonymizer() {
+		return new NullSynonymizer();
+	}
+
+	/* LanguageDetector */
+
+	/**
+	 * @since 0.1
+	 *
+	 * @return NullLanguageDetector
+	 */
+	public function newNullLanguageDetector() {
+		return new NullLanguageDetector();
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @return TextCatLanguageDetector
+	 */
+	public function newTextCatLanguageDetector() {
+		return new TextCatLanguageDetector();
+	}
+
+	/* Tokenizer */
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param string $text
+	 * @param string|null $languageCode
+	 *
+	 * @return Tokenizer
+	 */
+	public function newPreferredTokenizerByLanguage( $text, $languageCode = null ) {
+		$tokenizer = $this->newIcuWordBoundaryTokenizer();
+
+		if ( !$tokenizer->isAvailable() && CharacterExaminer::contains( CharacterExaminer::CJK_UNIFIED, $text ) ) {
+			return $this->newCJKMatchableTokenizer( $text );
+		} elseif ( !$tokenizer->isAvailable() ) {
+			return $this->newGenericRegExTokenizer( $tokenizer );
+		}
+
+		$tokenizer->setLocale( $languageCode );
+
+		$tokenizer->setWordTokenizerAttribute(
+			!CharacterExaminer::contains( CharacterExaminer::CJK_UNIFIED, $text )
+		);
+
+		return $this->newGenericRegExTokenizer( $tokenizer );
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param string $text
+	 *
+	 * @return Tokenizer
+	 */
+	public function newCJKMatchableTokenizer( $text ) {
+		$tokenizer = null;
+
+		if ( CharacterExaminer::contains( CharacterExaminer::HIRAGANA_KATAKANA, $text ) ) {
+			$tokenizer = $this->newJaTinySegmenterTokenizer();
+		} else {
+			$tokenizer = $this->newNGramTokenizer( $tokenizer );
+		}
+
+		$tokenizer = $this->newCJKSimpleCharacterRegExTokenizer( $tokenizer );
+
+		return $this->newGenericRegExTokenizer( $tokenizer );
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param Tokenizer|null $tokenizer
+	 *
+	 * @return Tokenizer
+	 */
+	public function newIcuWordBoundaryTokenizer( ?Tokenizer $tokenizer = null ) {
+		return new IcuWordBoundaryTokenizer( $tokenizer );
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param Tokenizer|null $tokenizer
+	 *
+	 * @return Tokenizer
+	 */
+	public function newGenericRegExTokenizer( ?Tokenizer $tokenizer = null ) {
+		return new GenericRegExTokenizer( $tokenizer );
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param Tokenizer|null $tokenizer
+	 *
+	 * @return Tokenizer
+	 */
+	public function newPunctuationRegExTokenizer( ?Tokenizer $tokenizer = null ) {
+		return new PunctuationRegExTokenizer( $tokenizer );
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @return Tokenizer
+	 */
+	public function newJaCompoundGroupTokenizer( ?Tokenizer $tokinizer = null ) {
+		return new JaCompoundGroupTokenizer( $tokinizer );
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @return Tokenizer
+	 */
+	public function newJaTinySegmenterTokenizer( ?Tokenizer $tokinizer = null ) {
+		return new JaTinySegmenterTokenizer( $tokinizer );
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @return Tokenizer
+	 */
+	public function newCJKSimpleCharacterRegExTokenizer( ?Tokenizer $tokinizer = null ) {
+		return new CJKSimpleCharacterRegExTokenizer( $tokinizer );
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @return Tokenizer
+	 */
+	public function newNGramTokenizer( ?Tokenizer $tokinizer = null, $ngram = 2 ) {
+		return new NGramTokenizer( $tokinizer, $ngram );
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/src/StopwordAnalyzer/ArrayStopwordAnalyzer.php

@@ -0,0 +1,45 @@
+<?php
+
+namespace Onoi\Tesa\StopwordAnalyzer;
+
+/**
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class ArrayStopwordAnalyzer implements StopwordAnalyzer {
+
+	/**
+	 * Any change to the content of its data files should be reflected in a
+	 * version change (the version number does not necessarily correlate with
+	 * the library version)
+	 */
+	const VERSION = '0.1';
+
+	/**
+	 * @var Cdb
+	 */
+	private $stopwords;
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param array $stopwords
+	 */
+	public function __construct( array $stopwords = [] ) {
+		$this->stopwords = array_flip( $stopwords );
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param string $word
+	 *
+	 * @return bool
+	 */
+	public function isStopWord( $word ) {
+		return isset( $this->stopwords[$word] );
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/src/StopwordAnalyzer/CdbStopwordAnalyzer.php

@@ -0,0 +1,124 @@
+<?php
+
+namespace Onoi\Tesa\StopwordAnalyzer;
+
+use Cdb\Reader;
+use Cdb\Writer;
+use Exception;
+use RuntimeException;
+
+/**
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class CdbStopwordAnalyzer implements StopwordAnalyzer {
+
+	/**
+	 * Any change to the content of its data files should be reflected in a
+	 * version change (the version number does not necessarily correlate with
+	 * the library version)
+	 */
+	const VERSION = '0.1.cdb';
+
+	/**
+	 * @var Cdb
+	 */
+	private $cdb;
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param string $target
+	 */
+	public function __construct( $target ) {
+		try {
+			$this->cdb = Reader::open( $target );
+		} catch ( Exception $e ) {
+			// Do nothing
+		}
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @return bool
+	 */
+	public function isAvailable() {
+		return $this->cdb !== null;
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param string $language
+	 *
+	 * @return string
+	 */
+	public static function getLocation() {
+		return str_replace( [ '\\', '/' ], DIRECTORY_SEPARATOR, __DIR__ . '/data/' );
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param string $language
+	 *
+	 * @return string
+	 */
+	public static function getTargetByLanguage( $language ) {
+		return self::getLocation() . 'cdb/' . strtolower( $language ?? '' ) . '.cdb';
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param string $word
+	 *
+	 * @return bool
+	 */
+	public function isStopWord( $word ) {
+		if ( $this->cdb !== null && $this->cdb->get( $word ) !== false ) {
+			return true;
+		}
+
+		return false;
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param string $location
+	 * @param string $language
+	 *
+	 * @return bool
+	 */
+	public static function createCdbByLanguage( $location, $language ) {
+		$language = strtolower( $language ?? '' );
+		$source = $location . $language . '.json';
+
+		if ( !file_exists( $source ) ) {
+			throw new RuntimeException( "{$source} is not available." );
+		}
+
+		$contents = json_decode( file_get_contents( $source ), true );
+
+		if ( !isset( $contents['list'] ) ) {
+			throw new RuntimeException( "JSON is missing the `list` index." );
+		}
+
+		$writer = Writer::open(
+			self::getTargetByLanguage( $language )
+		);
+
+		foreach ( $contents['list'] as $words ) {
+			$writer->set( trim( $words ), true );
+		}
+
+		$writer->close();
+
+		return true;
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/src/StopwordAnalyzer/NullStopwordAnalyzer.php

@@ -0,0 +1,24 @@
+<?php
+
+namespace Onoi\Tesa\StopwordAnalyzer;
+
+/**
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class NullStopwordAnalyzer implements StopwordAnalyzer {
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param string $word
+	 *
+	 * @return bool
+	 */
+	public function isStopWord( $word ) {
+		return false;
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/src/StopwordAnalyzer/README.md

@@ -0,0 +1,40 @@
+# StopwordAnalyzer
+
+This interface provides the means to filter specific frequent words (or characters) from a text corpus.
+
+```php
+
+use Onoi\Tesa\SanitizerFactory;
+
+$sanitizerFactory = new SanitizerFactory();
+
+$stopwordAnalyzer = $sanitizerFactory->newStopwordAnalyzerByLanguage(
+	'en'
+);
+
+$stopwordAnalyzer->isStopWord( 'foo' );
+
+```
+
+## CdbStopwordAnalyzer
+
+`CdbStopwordAnalyzer` uses [cdb][cdb] as storage backend to allow for an instant access to a list of
+stopwords on a per language basis.
+
+Adding a new set of stopwords to a language only requires to place a JSON file into the the `/data` folder
+and extend the `CdbStopwordAnalyzerTest` which ensures that listed languages and their JSON files are
+converted into a corresponding cdb file.
+
+```
+{
+	"@source": "...",
+	"version": "0.1",
+	"list":[
+		"a",
+		"about",
+		"..."
+	]
+}
+```
+
+[cdb]: https://en.wikipedia.org/wiki/Cdb_(software)

mediawiki/extensions/SemanticMediaWiki/Tesa/src/StopwordAnalyzer/StopwordAnalyzer.php

@@ -0,0 +1,22 @@
+<?php
+
+namespace Onoi\Tesa\StopwordAnalyzer;
+
+/**
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+interface StopwordAnalyzer {
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param string $word
+	 *
+	 * @return bool
+	 */
+	public function isStopWord( $word );
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/src/StopwordAnalyzer/data/de.json

@@ -0,0 +1,366 @@
+{
+	"@source": "http://www.ranks.nl/stopwords/german, https://github.com/SUL-DIG/hcrc_solr/stopwords_de.txt",
+	"version": "0.1",
+	"list":[
+		"aber",
+		"alle",
+		"allem",
+		"allen",
+		"aller",
+		"alles",
+		"als",
+		"also",
+		"am",
+		"an",
+		"ander",
+		"andere",
+		"anderem",
+		"anderen",
+		"anderer",
+		"anderes",
+		"anderm",
+		"andern",
+		"anderr",
+		"anders",
+		"auch",
+		"auf",
+		"aus",
+		"bei",
+		"bin",
+		"bis",
+		"bist",
+		"da",
+		"damit",
+		"dann",
+		"das",
+		"dasselbe",
+		"dazu",
+		"daß",
+		"dein",
+		"deine",
+		"deinem",
+		"deinen",
+		"deiner",
+		"deines",
+		"dem",
+		"demselben",
+		"den",
+		"denn",
+		"denselben",
+		"der",
+		"derer",
+		"derselbe",
+		"derselben",
+		"des",
+		"desselben",
+		"dessen",
+		"dich",
+		"die",
+		"dies",
+		"diese",
+		"dieselbe",
+		"dieselben",
+		"diesem",
+		"diesen",
+		"dieser",
+		"dieses",
+		"dir",
+		"doch",
+		"dort",
+		"du",
+		"durch",
+		"ein",
+		"eine",
+		"einem",
+		"einen",
+		"einer",
+		"eines",
+		"einig",
+		"einige",
+		"einigem",
+		"einigen",
+		"einiger",
+		"einiges",
+		"einmal",
+		"er",
+		"es",
+		"etwas",
+		"euch",
+		"euer",
+		"eure",
+		"eurem",
+		"euren",
+		"eurer",
+		"eures",
+		"für",
+		"gegen",
+		"gewesen",
+		"hab",
+		"habe",
+		"haben",
+		"hat",
+		"hatte",
+		"hatten",
+		"hier",
+		"hin",
+		"hinter",
+		"ich",
+		"ihm",
+		"ihn",
+		"ihnen",
+		"ihr",
+		"ihre",
+		"ihrem",
+		"ihren",
+		"ihrer",
+		"ihres",
+		"im",
+		"in",
+		"indem",
+		"ins",
+		"ist",
+		"jede",
+		"jedem",
+		"jeden",
+		"jeder",
+		"jedes",
+		"jene",
+		"jenem",
+		"jenen",
+		"jener",
+		"jenes",
+		"jetzt",
+		"kann",
+		"kein",
+		"keine",
+		"keinem",
+		"keinen",
+		"keiner",
+		"keines",
+		"können",
+		"könnte",
+		"machen",
+		"man",
+		"manche",
+		"manchem",
+		"manchen",
+		"mancher",
+		"manches",
+		"mein",
+		"meine",
+		"meinem",
+		"meinen",
+		"meiner",
+		"meines",
+		"mich",
+		"mir",
+		"mit",
+		"muss",
+		"musste",
+		"nach",
+		"nicht",
+		"nichts",
+		"noch",
+		"nun",
+		"nur",
+		"ob",
+		"oder",
+		"ohne",
+		"sehr",
+		"sein",
+		"seine",
+		"seinem",
+		"seinen",
+		"seiner",
+		"seines",
+		"selbst",
+		"sich",
+		"sie",
+		"sind",
+		"so",
+		"solche",
+		"solchem",
+		"solchen",
+		"solcher",
+		"solches",
+		"soll",
+		"sollte",
+		"sondern",
+		"sonst",
+		"um",
+		"und",
+		"uns",
+		"unse",
+		"unsem",
+		"unsen",
+		"unser",
+		"unses",
+		"unter",
+		"viel",
+		"vom",
+		"von",
+		"vor",
+		"war",
+		"waren",
+		"warst",
+		"was",
+		"weg",
+		"weil",
+		"weiter",
+		"welche",
+		"welchem",
+		"welchen",
+		"welcher",
+		"welches",
+		"wenn",
+		"werde",
+		"werden",
+		"wie",
+		"wieder",
+		"will",
+		"wir",
+		"wird",
+		"wirst",
+		"wo",
+		"wollen",
+		"wollte",
+		"während",
+		"würde",
+		"würden",
+		"zu",
+		"zum",
+		"zur",
+		"zwar",
+		"zwischen",
+		"über",
+		"aber",
+		"als",
+		"am",
+		"an",
+		"auch",
+		"auf",
+		"aus",
+		"bei",
+		"bin",
+		"bis",
+		"bist",
+		"da",
+		"dadurch",
+		"daher",
+		"darum",
+		"das",
+		"dass",
+		"daß",
+		"dein",
+		"deine",
+		"dem",
+		"den",
+		"der",
+		"des",
+		"deshalb",
+		"dessen",
+		"die",
+		"dies",
+		"dieser",
+		"dieses",
+		"doch",
+		"dort",
+		"du",
+		"durch",
+		"ein",
+		"eine",
+		"einem",
+		"einen",
+		"einer",
+		"eines",
+		"er",
+		"es",
+		"euer",
+		"eure",
+		"für",
+		"hatte",
+		"hatten",
+		"hattest",
+		"hattet",
+		"hier",
+		"hinter",
+		"ich",
+		"ihr",
+		"ihre",
+		"im",
+		"in",
+		"ist",
+		"ja",
+		"jede",
+		"jedem",
+		"jeden",
+		"jeder",
+		"jedes",
+		"jener",
+		"jenes",
+		"jetzt",
+		"kann",
+		"kannst",
+		"können",
+		"könnt",
+		"machen",
+		"mein",
+		"meine",
+		"mit",
+		"musst",
+		"muß",
+		"mußt",
+		"müssen",
+		"müßt",
+		"nach",
+		"nachdem",
+		"nein",
+		"nicht",
+		"nun",
+		"oder",
+		"seid",
+		"sein",
+		"seine",
+		"sich",
+		"sie",
+		"sind",
+		"soll",
+		"sollen",
+		"sollst",
+		"sollt",
+		"sonst",
+		"soweit",
+		"sowie",
+		"und",
+		"unser",
+		"unsere",
+		"unter",
+		"vom",
+		"von",
+		"vor",
+		"wann",
+		"warum",
+		"was",
+		"weiter",
+		"weitere",
+		"wenn",
+		"wer",
+		"werde",
+		"werden",
+		"werdet",
+		"weshalb",
+		"wie",
+		"wieder",
+		"wieso",
+		"wir",
+		"wird",
+		"wirst",
+		"wo",
+		"woher",
+		"wohin",
+		"zu",
+		"zum",
+		"zur",
+		"über"
+	]
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/src/StopwordAnalyzer/data/es.json

@@ -0,0 +1,185 @@
+{
+	"@source": "http://www.ranks.nl/stopwords/spanish",
+	"version": "0.1",
+	"list":[
+		"alguna",
+		"algunas",
+		"alguno",
+		"algunos",
+		"algún",
+		"ambos",
+		"ampleamos",
+		"ante",
+		"antes",
+		"aquel",
+		"aquellas",
+		"aquellos",
+		"aqui",
+		"arriba",
+		"atras",
+		"bajo",
+		"bastante",
+		"bien",
+		"cada",
+		"cierta",
+		"ciertas",
+		"cierto",
+		"ciertos",
+		"como",
+		"con",
+		"conseguimos",
+		"conseguir",
+		"consigo",
+		"consigue",
+		"consiguen",
+		"consigues",
+		"cual",
+		"cuando",
+		"dentro",
+		"desde",
+		"donde",
+		"dos",
+		"el",
+		"ellas",
+		"ellos",
+		"empleais",
+		"emplean",
+		"emplear",
+		"empleas",
+		"empleo",
+		"en",
+		"encima",
+		"entonces",
+		"entre",
+		"era",
+		"eramos",
+		"eran",
+		"eras",
+		"eres",
+		"es",
+		"esta",
+		"estaba",
+		"estado",
+		"estais",
+		"estamos",
+		"estan",
+		"estoy",
+		"fin",
+		"fue",
+		"fueron",
+		"fui",
+		"fuimos",
+		"gueno",
+		"ha",
+		"hace",
+		"haceis",
+		"hacemos",
+		"hacen",
+		"hacer",
+		"haces",
+		"hago",
+		"incluso",
+		"intenta",
+		"intentais",
+		"intentamos",
+		"intentan",
+		"intentar",
+		"intentas",
+		"intento",
+		"ir",
+		"la",
+		"largo",
+		"las",
+		"lo",
+		"los",
+		"mientras",
+		"mio",
+		"modo",
+		"muchos",
+		"muy",
+		"nos",
+		"nosotros",
+		"otro",
+		"para",
+		"pero",
+		"podeis",
+		"podemos",
+		"poder",
+		"podria",
+		"podriais",
+		"podriamos",
+		"podrian",
+		"podrias",
+		"por",
+		"por",
+		"porque",
+		"primero",
+		"puede",
+		"pueden",
+		"puedo",
+		"quien",
+		"qué",
+		"sabe",
+		"sabeis",
+		"sabemos",
+		"saben",
+		"saber",
+		"sabes",
+		"ser",
+		"si",
+		"siendo",
+		"sin",
+		"sobre",
+		"sois",
+		"solamente",
+		"solo",
+		"somos",
+		"soy",
+		"su",
+		"sus",
+		"también",
+		"teneis",
+		"tenemos",
+		"tener",
+		"tengo",
+		"tiempo",
+		"tiene",
+		"tienen",
+		"todo",
+		"trabaja",
+		"trabajais",
+		"trabajamos",
+		"trabajan",
+		"trabajar",
+		"trabajas",
+		"trabajo",
+		"tras",
+		"tuyo",
+		"ultimo",
+		"un",
+		"una",
+		"unas",
+		"uno",
+		"unos",
+		"usa",
+		"usais",
+		"usamos",
+		"usan",
+		"usar",
+		"usas",
+		"uso",
+		"va",
+		"vais",
+		"valor",
+		"vamos",
+		"van",
+		"vaya",
+		"verdad",
+		"verdadera",
+		"verdadero",
+		"vosotras",
+		"vosotros",
+		"voy",
+		"yo"
+	]
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/src/StopwordAnalyzer/data/fr.json

@@ -0,0 +1,122 @@
+{
+	"@source": "http://www.ranks.nl/stopwords/french",
+	"version": "0.1",
+	"list":[
+		"alors",
+		"au",
+		"aucuns",
+		"aussi",
+		"autre",
+		"avant",
+		"avec",
+		"avoir",
+		"bon",
+		"car",
+		"ce",
+		"cela",
+		"ces",
+		"ceux",
+		"chaque",
+		"ci",
+		"comme",
+		"comment",
+		"dans",
+		"dedans",
+		"dehors",
+		"depuis",
+		"des",
+		"devrait",
+		"doit",
+		"donc",
+		"dos",
+		"du",
+		"début",
+		"elle",
+		"elles",
+		"en",
+		"encore",
+		"essai",
+		"est",
+		"et",
+		"eu",
+		"fait",
+		"faites",
+		"fois",
+		"font",
+		"hors",
+		"ici",
+		"il",
+		"ils",
+		"je",
+		"juste",
+		"la",
+		"le",
+		"les",
+		"leur",
+		"là",
+		"ma",
+		"maintenant",
+		"mais",
+		"mes",
+		"mine",
+		"moins",
+		"mon",
+		"mot",
+		"même",
+		"ni",
+		"nommés",
+		"notre",
+		"nous",
+		"ou",
+		"où",
+		"par",
+		"parce",
+		"pas",
+		"peu",
+		"peut",
+		"plupart",
+		"pour",
+		"pourquoi",
+		"quand",
+		"que",
+		"quel",
+		"quelle",
+		"quelles",
+		"quels",
+		"qui",
+		"sa",
+		"sans",
+		"ses",
+		"seulement",
+		"si",
+		"sien",
+		"son",
+		"sont",
+		"sous",
+		"soyez",
+		"sujet",
+		"sur",
+		"ta",
+		"tandis",
+		"tellement",
+		"tels",
+		"tes",
+		"ton",
+		"tous",
+		"tout",
+		"trop",
+		"très",
+		"tu",
+		"voient",
+		"vont",
+		"votre",
+		"vous",
+		"vu",
+		"ça",
+		"étaient",
+		"état",
+		"étions",
+		"été",
+		"être"
+	]
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/src/StopwordAnalyzer/data/ja.json

@@ -0,0 +1,115 @@
+{
+	"@source": "https://github.com/SUL-DIG/hcrc_solr/blob/master/conf/stopwords_all_lang.txt, stopwords_ja.txt",
+	"version": "0.1",
+	"list":[
+		"あっ",
+		"あり",
+		"ある",
+		"い",
+		"いう",
+		"いる",
+		"う",
+		"うち",
+		"お",
+		"および",
+		"おり",
+		"か",
+		"かつて",
+		"から",
+		"が",
+		"き",
+		"ここ",
+		"こと",
+		"この",
+		"これ",
+		"これら",
+		"さ",
+		"さらに",
+		"し",
+		"しかし",
+		"する",
+		"ず",
+		"せ",
+		"せる",
+		"そして",
+		"その",
+		"その他",
+		"その後",
+		"それ",
+		"それぞれ",
+		"た",
+		"ただし",
+		"たち",
+		"ため",
+		"たり",
+		"だ",
+		"だっ",
+		"つ",
+		"て",
+		"で",
+		"でき",
+		"できる",
+		"です",
+		"では",
+		"でも",
+		"と",
+		"という",
+		"といった",
+		"とき",
+		"ところ",
+		"として",
+		"とともに",
+		"とも",
+		"と共に",
+		"な",
+		"ない",
+		"なお",
+		"なかっ",
+		"ながら",
+		"なく",
+		"なっ",
+		"など",
+		"なら",
+		"なり",
+		"なる",
+		"に",
+		"において",
+		"における",
+		"について",
+		"にて",
+		"によって",
+		"により",
+		"による",
+		"に対して",
+		"に対する",
+		"に関する",
+		"の",
+		"ので",
+		"のみ",
+		"は",
+		"ば",
+		"へ",
+		"ほか",
+		"ほとんど",
+		"ほど",
+		"ます",
+		"また",
+		"または",
+		"まで",
+		"も",
+		"もの",
+		"ものの",
+		"や",
+		"よう",
+		"より",
+		"ら",
+		"られ",
+		"られる",
+		"れ",
+		"れる",
+		"を",
+		"ん",
+		"及び",
+		"特に"
+	]
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/src/StopwordAnalyzer/data/pt-br.json

@@ -0,0 +1,132 @@
+{
+	"@source": "http://www.ranks.nl/stopwords/brazilian",
+	"version": "0.1",
+	"list": [
+		"a",
+		"ainda",
+		"alem",
+		"ambas",
+		"ambos",
+		"antes",
+		"ao",
+		"aonde",
+		"aos",
+		"apos",
+		"aquele",
+		"aqueles",
+		"as",
+		"assim",
+		"com",
+		"como",
+		"contra",
+		"contudo",
+		"cuja",
+		"cujas",
+		"cujo",
+		"cujos",
+		"da",
+		"das",
+		"de",
+		"dela",
+		"dele",
+		"deles",
+		"demais",
+		"depois",
+		"desde",
+		"desta",
+		"deste",
+		"dispoe",
+		"dispoem",
+		"diversa",
+		"diversas",
+		"diversos",
+		"do",
+		"dos",
+		"durante",
+		"e",
+		"ela",
+		"elas",
+		"ele",
+		"eles",
+		"em",
+		"entao",
+		"entre",
+		"essa\tessas",
+		"esse",
+		"esses",
+		"esta",
+		"estas",
+		"este",
+		"estes",
+		"ha",
+		"isso",
+		"isto",
+		"logo",
+		"mais",
+		"mas",
+		"mediante",
+		"menos",
+		"mesma",
+		"mesmas",
+		"mesmo",
+		"mesmos",
+		"na",
+		"nao",
+		"nas",
+		"nas",
+		"nem",
+		"nesse",
+		"neste",
+		"nos",
+		"o",
+		"os",
+		"ou",
+		"outra",
+		"outras",
+		"outro",
+		"outros",
+		"pelas",
+		"pelas",
+		"pelo",
+		"pelos",
+		"perante",
+		"pois",
+		"por",
+		"porque",
+		"portanto",
+		"propios",
+		"proprio",
+		"quais",
+		"qual",
+		"qualquer",
+		"quando",
+		"quanto\tque",
+		"quem",
+		"quer",
+		"se",
+		"seja",
+		"sem",
+		"sendo",
+		"seu",
+		"seus",
+		"sob",
+		"sobre",
+		"sua",
+		"suas",
+		"tal",
+		"tambem",
+		"teu",
+		"teus",
+		"toda",
+		"todas",
+		"todo",
+		"todos",
+		"tua",
+		"tuas",
+		"tudo",
+		"um",
+		"uma",
+		"umas",
+		"uns"
+	]
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/src/StopwordAnalyzer/data/pt.json

@@ -0,0 +1,151 @@
+{
+	"@source": "http://www.ranks.nl/stopwords/portugese",
+	"version": "0.1",
+	"list": [
+		"último",
+		"é",
+		"acerca",
+		"agora",
+		"algmas",
+		"alguns",
+		"ali",
+		"ambos",
+		"antes",
+		"apontar",
+		"aquela",
+		"aquelas",
+		"aquele",
+		"aqueles",
+		"aqui",
+		"atrás",
+		"bem",
+		"bom",
+		"cada",
+		"caminho",
+		"cima",
+		"com",
+		"como",
+		"comprido",
+		"conhecido",
+		"corrente",
+		"das",
+		"debaixo",
+		"dentro",
+		"desde",
+		"desligado",
+		"deve",
+		"devem",
+		"deverá",
+		"direita",
+		"diz",
+		"dizer",
+		"dois",
+		"dos",
+		"e",
+		"ela",
+		"ele",
+		"eles",
+		"em",
+		"enquanto",
+		"então",
+		"está",
+		"estão",
+		"estado",
+		"estar\testará",
+		"este",
+		"estes",
+		"esteve",
+		"estive",
+		"estivemos",
+		"estiveram",
+		"eu",
+		"fará",
+		"faz",
+		"fazer",
+		"fazia",
+		"fez",
+		"fim",
+		"foi",
+		"fora",
+		"horas",
+		"iniciar",
+		"inicio",
+		"ir",
+		"irá",
+		"ista",
+		"iste",
+		"isto",
+		"ligado",
+		"maioria",
+		"maiorias",
+		"mais",
+		"mas",
+		"mesmo",
+		"meu",
+		"muito",
+		"muitos",
+		"nós",
+		"não",
+		"nome",
+		"nosso",
+		"novo",
+		"o",
+		"onde",
+		"os",
+		"ou",
+		"outro",
+		"para",
+		"parte",
+		"pegar",
+		"pelo",
+		"pessoas",
+		"pode",
+		"poderá\tpodia",
+		"por",
+		"porque",
+		"povo",
+		"promeiro",
+		"quê",
+		"qual",
+		"qualquer",
+		"quando",
+		"quem",
+		"quieto",
+		"são",
+		"saber",
+		"sem",
+		"ser",
+		"seu",
+		"somente",
+		"têm",
+		"tal",
+		"também",
+		"tem",
+		"tempo",
+		"tenho",
+		"tentar",
+		"tentaram",
+		"tente",
+		"tentei",
+		"teu",
+		"teve",
+		"tipo",
+		"tive",
+		"todos",
+		"trabalhar",
+		"trabalho",
+		"tu",
+		"um",
+		"uma",
+		"umas",
+		"uns",
+		"usa",
+		"usar",
+		"valor",
+		"veja",
+		"ver",
+		"verdade",
+		"verdadeiro",
+		"você"
+	]
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/src/StopwordAnalyzer/data/zh.json

@@ -0,0 +1,5 @@
+{
+	"@source": "https://raw.githubusercontent.com/6/stopwords-json/master/dist/zh.json",
+	"version": "0.1",
+	"list":["、","。","〈","〉","《","》","一","一切","一则","一方面","一旦","一来","一样","一般","七","万一","三","上下","不仅","不但","不光","不单","不只","不如","不怕","不惟","不成","不拘","不比","不然","不特","不独","不管","不论","不过","不问","与","与其","与否","与此同时","且","两者","个","临","为","为了","为什么","为何","为着","乃","乃至","么","之","之一","之所以","之类","乌乎","乎","乘","九","也","也好","也罢","了","二","于","于是","于是乎","云云","五","人家","什么","什么样","从","从而","他","他人","他们","以","以便","以免","以及","以至","以至于","以致","们","任","任何","任凭","似的","但","但是","何","何况","何处","何时","作为","你","你们","使得","例如","依","依照","俺","俺们","倘","倘使","倘或","倘然","倘若","借","假使","假如","假若","像","八","六","兮","关于","其","其一","其中","其二","其他","其余","其它","其次","具体地说","具体说来","再者","再说","冒","冲","况且","几","几时","凭","凭借","则","别","别的","别说","到","前后","前者","加之","即","即令","即使","即便","即或","即若","又","及","及其","及至","反之","反过来","反过来说","另","另一方面","另外","只是","只有","只要","只限","叫","叮咚","可","可以","可是","可见","各","各个","各位","各种","各自","同","同时","向","向着","吓","吗","否则","吧","吧哒","吱","呀","呃","呕","呗","呜","呜呼","呢","呵","呸","呼哧","咋","和","咚","咦","咱","咱们","咳","哇","哈","哈哈","哉","哎","哎呀","哎哟","哗","哟","哦","哩","哪","哪个","哪些","哪儿","哪天","哪年","哪怕","哪样","哪边","哪里","哼","哼唷","唉","啊","啐","啥","啦","啪达","喂","喏","喔唷","嗡嗡","嗬","嗯","嗳","嘎","嘎登","嘘","嘛","嘻","嘿","四","因","因为","因此","因而","固然","在","在下","地","多","多少","她","她们","如","如上所述","如何","如其","如果","如此","如若","宁","宁可","宁愿","宁肯","它","它们","对","对于","将","尔后","尚且","就","就是","就是说","尽","尽管","岂但","己","并","并且","开外","开始","归","当","当着","彼","彼此","往","待","得","怎","怎么","怎么办","怎么样","怎样","总之","总的来看","总的来说","总的说来","总而言之","恰恰相反","您","慢说","我","我们","或","或是","或者","所","所以","打","把","抑或","拿","按","按照","换句话说","换言之","据","接着","故","故此","旁人","无宁","无论","既","既是","既然","时候","是","是的","替","有","有些","有关","有的","望","朝","朝着","本","本着","来","来着","极了","果然","果真","某","某个","某些","根据","正如","此","此外","此间","毋宁","每","每当","比","比如","比方","沿","沿着","漫说","焉","然则","然后","然而","照","照着","甚么","甚而","甚至","用","由","由于","由此可见","的","的话","相对而言","省得","着","着呢","矣","离","第","等","等等","管","紧接着","纵","纵令","纵使","纵然","经","经过","结果","给","继而","综上所述","罢了","者","而","而且","而况","而外","而已","而是","而言","能","腾","自","自个儿","自从","自各儿","自家","自己","自身","至","至于","若","若是","若非","莫若","虽","虽则","虽然","虽说","被","要","要不","要不是","要不然","要么","要是","让","论","设使","设若","该","诸位","谁","谁知","赶","起","起见","趁","趁着","越是","跟","较","较之","边","过","还是","还有","这","这个","这么","这么些","这么样","这么点儿","这些","这会儿","这儿","这就是说","这时","这样","这边","这里","进而","连","连同","通过","遵照","那","那个","那么","那么些","那么样","那些","那会儿","那儿","那时","那样","那边","那里","鄙人","鉴于","阿","除","除了","除此之外","除非","随","随着","零","非但","非徒","靠","顺","顺着","首先","︿","！","＃","＄","％","＆","（","）","＊","＋","，","０","１","２","３","４","５","６","７","８","９","：","；","＜","＞","？","＠","［","］","｛","｜","｝","～","￥"]
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/src/Synonymizer/NullSynonymizer.php

@@ -0,0 +1,24 @@
+<?php
+
+namespace Onoi\Tesa\Synonymizer;
+
+/**
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class NullSynonymizer implements Synonymizer {
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param string $word
+	 *
+	 * @return string
+	 */
+	public function synonymize( $word ) {
+		return $word;
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/src/Synonymizer/Synonymizer.php

@@ -0,0 +1,22 @@
+<?php
+
+namespace Onoi\Tesa\Synonymizer;
+
+/**
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+interface Synonymizer {
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param string $word
+	 *
+	 * @return string
+	 */
+	public function synonymize( $word );
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/src/Tokenizer/CJKSimpleCharacterRegExTokenizer.php

@@ -0,0 +1,82 @@
+<?php
+
+namespace Onoi\Tesa\Tokenizer;
+
+/**
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class CJKSimpleCharacterRegExTokenizer implements Tokenizer {
+
+	/**
+	 * @var Tokenizer
+	 */
+	private $tokenizer;
+
+	/**
+	 * @var string
+	 */
+	private $patternExemption = '';
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param Tokenizer|null $tokenizer
+	 */
+	public function __construct( ?Tokenizer $tokenizer = null ) {
+		$this->tokenizer = $tokenizer;
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * {@inheritDoc}
+	 */
+	public function setOption( $name, $value ) {
+		if ( $this->tokenizer !== null ) {
+			$this->tokenizer->setOption( $name, $value );
+		}
+
+		if ( $name === self::REGEX_EXEMPTION ) {
+			$this->patternExemption = $value;
+		}
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * {@inheritDoc}
+	 */
+	public function isWordTokenizer() {
+		return false;
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * {@inheritDoc}
+	 */
+	public function tokenize( $string ) {
+		if ( $this->tokenizer !== null ) {
+			$string = implode( " ", $this->tokenizer->tokenize( $string ) );
+		}
+
+		// Filter is based on https://github.com/kitech/cms-drupal/blob/master/modules/csplitter/filter.txt
+		$pattern = str_replace(
+			$this->patternExemption,
+			'',
+			'([\s\、，,。／？《》〈〉；：“”＂〃＇｀［］｛｝＼｜～！－＝＿＋）（()＊…—─％￥…◆★◇□■【】＃·啊吧把并被才从的得当对但到地而该过个给还和叫将就可来了啦里没你您哪那呢去却让使是时省随他我为现县向像象要由矣已以也又与于在之这则最乃\/\(\)\[\]{}<>\r\n"]|(?<!\d)\.(?!\d))'
+		);
+
+		$result = preg_split( '/' . $pattern . '/u', $string, -1, PREG_SPLIT_NO_EMPTY );
+
+		if ( $result !== false ) {
+			return $result;
+		}
+
+		return [];
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/src/Tokenizer/GenericRegExTokenizer.php

@@ -0,0 +1,85 @@
+<?php
+
+namespace Onoi\Tesa\Tokenizer;
+
+/**
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class GenericRegExTokenizer implements Tokenizer {
+
+	/**
+	 * @var Tokenizer
+	 */
+	private $tokenizer;
+
+	/**
+	 * @var string
+	 */
+	private $patternExemption = '';
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param Tokenizer|null $tokenizer
+	 */
+	public function __construct( ?Tokenizer $tokenizer = null ) {
+		$this->tokenizer = $tokenizer;
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * {@inheritDoc}
+	 */
+	public function setOption( $name, $value ) {
+		if ( $this->tokenizer !== null ) {
+			$this->tokenizer->setOption( $name, $value );
+		}
+
+		if ( $name === self::REGEX_EXEMPTION ) {
+			$this->patternExemption = $value;
+		}
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * {@inheritDoc}
+	 */
+	public function isWordTokenizer() {
+		return $this->tokenizer !== null ? $this->tokenizer->isWordTokenizer() : true;
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param string $string
+	 *
+	 * @return array|false
+	 */
+	public function tokenize( $string ) {
+		if ( $this->tokenizer !== null ) {
+			$string = implode( " ", $this->tokenizer->tokenize( $string ) );
+		}
+
+		// (?<=\p{L})(?=\p{N}) to split alphanumeric and numeric
+
+		$pattern = str_replace(
+			$this->patternExemption,
+			'',
+			'([\s\-_,:;?!%\'\|\/\(\)\[\]{}<>\r\n"]|(?<!\d)\.(?!\d)|(?<=\p{L})(?=\p{N}))'
+		);
+
+		$result = preg_split( '/' . $pattern . '/u', $string, -1, PREG_SPLIT_NO_EMPTY );
+
+		if ( $result === false ) {
+			$result = [];
+		}
+
+		return $result;
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/src/Tokenizer/IcuWordBoundaryTokenizer.php

@@ -0,0 +1,131 @@
+<?php
+
+namespace Onoi\Tesa\Tokenizer;
+
+use IntlRuleBasedBreakIterator;
+
+/**
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class IcuWordBoundaryTokenizer implements Tokenizer {
+
+	/**
+	 * @var Tokenizer
+	 */
+	private $tokenizer;
+
+	/**
+	 * @var string
+	 */
+	private $locale = 'en';
+
+	/**
+	 * @var string
+	 */
+	private $isWordTokenizer = true;
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param Tokenizer|null $tokenizer
+	 */
+	public function __construct( ?Tokenizer $tokenizer = null ) {
+		$this->tokenizer = $tokenizer;
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * {@inheritDoc}
+	 */
+	public function setOption( $name, $value ) {
+		if ( $this->tokenizer !== null ) {
+			$this->tokenizer->setOption( $name, $value );
+		}
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * {@inheritDoc}
+	 */
+	public function isWordTokenizer() {
+		return $this->isWordTokenizer;
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * {@inheritDoc}
+	 */
+	public function setWordTokenizerAttribute( $usesWordBoundaries ) {
+		return $this->isWordTokenizer = $usesWordBoundaries;
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @return bool
+	 */
+	public function isAvailable() {
+		return class_exists( 'IntlRuleBasedBreakIterator' );
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param string $locale
+	 */
+	public function setLocale( $locale ) {
+		$this->locale = $locale;
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param string $string
+	 *
+	 * @return array|false
+	 */
+	public function tokenize( $string ) {
+		if ( $this->tokenizer !== null ) {
+			$string = implode( " ", $this->tokenizer->tokenize( $string ) );
+		}
+
+		if ( !$this->isAvailable() ) {
+			return $this->tokenizer !== null ? $this->tokenizer->tokenize( $string ) : [ $string ];
+		}
+
+		return $this->createTokens( $string );
+	}
+
+	private function createTokens( $string ) {
+		$tokens = [];
+
+		if ( $tokenizer = IntlRuleBasedBreakIterator::createWordInstance( $this->locale ) ) {
+			$tokenizer->setText( $string );
+			$prev = 0;
+
+			foreach ( $tokenizer as $token ) {
+
+				if ( $token == 0 ) {
+					continue;
+				}
+
+				$res = substr( $string, $prev, $token - $prev );
+
+				if ( $res !== '' && $res !== ' ' ) {
+					$tokens[] = $res;
+				}
+
+				$prev = $token;
+			}
+		}
+
+		return $tokens;
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/src/Tokenizer/JaCompoundGroupTokenizer.php

@@ -0,0 +1,226 @@
+<?php
+
+namespace Onoi\Tesa\Tokenizer;
+
+use Onoi\Tesa\CharacterExaminer;
+
+/**
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class JaCompoundGroupTokenizer implements Tokenizer {
+
+	/**
+	 * @var array
+	 */
+	private $compound = [
+		"あっ",
+		"あり",
+		"ある",
+		"い",
+		"いう",
+		"いる",
+		"う",
+		"うち",
+		"お",
+		"および",
+		"おり",
+		"か",
+		"かつて",
+		"から",
+		"が",
+		"き",
+		"ここ",
+		"こと",
+		"この",
+		"これ",
+		"これら",
+		"さ",
+		"さらに",
+		"し",
+		"しかし",
+		"する",
+		"ず",
+		"せ",
+		"せる",
+		"そして",
+		"その",
+		"その他",
+		"その後",
+		"それ",
+		"それぞれ",
+		"た",
+		"ただし",
+		"たち",
+		"ため",
+		"たり",
+		"だ",
+		"だっ",
+		"つ",
+		"て",
+		"で",
+		"でき",
+		"できる",
+		"です",
+		"では",
+		"でも",
+		"と",
+		"という",
+		"といった",
+		"とき",
+		"ところ",
+		"として",
+		"とともに",
+		"とも",
+		"と共に",
+		"な",
+		"ない",
+		"なお",
+		"なかっ",
+		"ながら",
+		"なく",
+		"なっ",
+		"など",
+		"なら",
+		"なり",
+		"なる",
+		"に",
+		"において",
+		"における",
+		"について",
+		"にて",
+		"によって",
+		"により",
+		"による",
+		"に対して",
+		"に対する",
+		"に関する",
+		"の",
+		"ので",
+		"のみ",
+		"は",
+		"ば",
+		"へ",
+		"ほか",
+		"ほとんど",
+		"ほど",
+		"ます",
+		"また",
+		"または",
+		"まで",
+		"も",
+		"もの",
+		"ものの",
+		"や",
+		"よう",
+		"より",
+		"ら",
+		"られ",
+		"られる",
+		"れ",
+		"れる",
+		"を",
+		"ん",
+		"及び",
+		"特に",
+		"、",
+		"。",
+		"「",
+		"」"
+	];
+
+	/**
+	 * @var Tokenizer
+	 */
+	private $tokenizer;
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param Tokenizer|null $tokenizer
+	 */
+	public function __construct( ?Tokenizer $tokenizer = null ) {
+		$this->tokenizer = $tokenizer;
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * {@inheritDoc}
+	 */
+	public function setOption( $name, $value ) {
+		if ( $this->tokenizer !== null ) {
+			$this->tokenizer->setOption( $name, $value );
+		}
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * {@inheritDoc}
+	 */
+	public function isWordTokenizer() {
+		return false;
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * {@inheritDoc}
+	 */
+	public function tokenize( $string ) {
+		if ( $this->tokenizer !== null ) {
+			$string = implode( " ", $this->tokenizer->tokenize( $string ) );
+		}
+
+		$result = explode( " ", $this->splitByCharacterGroup(
+			str_replace( $this->compound, ' ', $string ) )
+		);
+
+		foreach ( $result as $key => $value ) {
+			if ( $value === '' ) {
+				unset( $result[$key] );
+			}
+
+			// Single katakana/hiragana are exempted
+			if ( mb_strlen( $value ) === 1 && CharacterExaminer::contains( CharacterExaminer::HIRAGANA_KATAKANA, $value ) ) {
+				unset( $result[$key] );
+			}
+		}
+
+		if ( $result !== false ) {
+			return array_values( $result );
+		}
+
+		return [];
+	}
+
+	/**
+	 * @see MediaWiki LanguageJa::segmentByWord
+	 *
+	 * @since 0.1
+	 *
+	 * {@inheritDoc}
+	 */
+	public function splitByCharacterGroup( $string ) {
+		// Space strings of like hiragana/katakana/kanji
+		$hiragana = '(?:\xe3(?:\x81[\x80-\xbf]|\x82[\x80-\x9f]))'; # U3040-309f
+		$katakana = '(?:\xe3(?:\x82[\xa0-\xbf]|\x83[\x80-\xbf]))'; # U30a0-30ff
+		$kanji = '(?:\xe3[\x88-\xbf][\x80-\xbf]'
+			. '|[\xe4-\xe8][\x80-\xbf]{2}'
+			. '|\xe9[\x80-\xa5][\x80-\xbf]'
+			. '|\xe9\xa6[\x80-\x99])';
+			# U3200-9999 = \xe3\x88\x80-\xe9\xa6\x99
+
+		$reg = "/({$hiragana}+|{$katakana}+|{$kanji}+)/";
+
+		return $this->insertSpace( $string, $reg );
+	}
+
+	private function insertSpace( $string, $pattern ) {
+		return preg_replace( '/ +/', ' ', preg_replace( $pattern, " $1 ", $string ) );
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/src/Tokenizer/JaTinySegmenterTokenizer.php

@@ -0,0 +1,262 @@
+<?php
+
+namespace Onoi\Tesa\Tokenizer;
+
+use RuntimeException;
+
+/**
+ * PHP Version of the TinySegmenter as a super compact Japanese tokenizer.
+ * - https://github.com/setchi/codeute/blob/71c09c86cd1ce1cf9c8ca4d20b1db60b3784227a/fuel/app/classes/model/lib/tiny_segmenter.php
+ *
+ * TinySegmenter was originally developed by Taku Kudo <taku(at)chasen.org>.
+ * Pulished under the BSD license http://chasen.org/~taku/software/TinySegmenter/LICENCE.txt
+ *
+ * PHP Version was developed by xnights <programming.magic(at)gmail.com>.
+ * For details, see http://programming-magic.com/?id=172
+ *
+ * The model is based on the http://research.nii.ac.jp/src/list.html corpus
+ * together with an optimized L1-norm regularization.
+ *
+ * - https://github.com/shogo82148/TinySegmenterMaker
+ *
+ * @since 0.1
+ */
+class JaTinySegmenterTokenizer implements Tokenizer {
+
+	private $patterns_ = [
+		"[一二三四五六七八九十百千万億兆]" => "M", // numbers (japanese)
+		"[一-龠々〆ヵヶ]" => "H", // kanji & misc characters
+		"[ぁ-ん]" => "I", // hiragana
+		"[ァ-ヴーｱ-ﾝﾞｰ]" => "K", // katakana
+		"[a-zA-Zａ-ｚＡ-Ｚ]" => "A", // ascii / romaji letters
+		"[0-9０-９]" => "N", // ascii / romaji numbers
+	];
+
+	/**
+	 * @var Tokenizer
+	 */
+	private $tokenizer;
+
+	/**
+	 * This is kept static on purpose.
+	 * @var array
+	 */
+	private static $model;
+
+	/**
+	 * @var string
+	 */
+	private $modelFile;
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param Tokenizer|null $tokenizer
+	 */
+	public function __construct( ?Tokenizer $tokenizer = null ) {
+		$this->tokenizer = $tokenizer;
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * {@inheritDoc}
+	 */
+	public function setOption( $name, $value ) {
+		if ( $this->tokenizer !== null ) {
+			$this->tokenizer->setOption( $name, $value );
+		}
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * {@inheritDoc}
+	 */
+	public function isWordTokenizer() {
+		return false;
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * {@inheritDoc}
+	 */
+	public function tokenize( $string ) {
+		if ( $this->tokenizer !== null ) {
+			$string = implode( " ", $this->tokenizer->tokenize( $string ) );
+		}
+
+		return $this->loadModel()->segment( $string );
+	}
+
+	private function loadModel() {
+		if ( self::$model !== null ) {
+			return $this;
+		}
+
+		$contents = null;
+		$file = __DIR__ . '/model/rwcp.model.json';
+
+		if ( ( $contents = @file_get_contents( $file ) ) !== false ) {
+			self::$model = json_decode( $contents, true );
+		}
+
+		if ( $contents === false || json_last_error() !== JSON_ERROR_NONE ) {
+			throw new RuntimeException( "Couldn't read the model from {$file}." );
+		}
+
+		return $this;
+	}
+
+	protected function segment( $input, $encoding = null ) {
+		if ( !$input ) {
+			return [];
+		}
+
+		if ( !$encoding ) {
+			$encoding = mb_detect_encoding( $input );
+		}
+
+		if ( $encoding !== 'UTF-8' ) {
+			$input = mb_convert_encoding( $input, 'UTF-8', $encoding );
+		}
+
+		$result = [];
+		$seg = [ "B3", "B2", "B1" ];
+
+		$ctype = [ "O", "O", "O" ];
+		$o = $this->mb_string_to_array_( $input );
+
+		for ( $i = 0; $i < count( $o ); ++$i ) {
+			$seg[] = $o[$i];
+			$ctype[] = $this->ctype_( $o[$i] );
+		}
+
+		$seg[] = "E1";
+		$seg[] = "E2";
+		$seg[] = "E3";
+		$ctype[] = "O";
+		$ctype[] = "O";
+		$ctype[] = "O";
+		$word = $seg[3];
+		$p1 = "U";
+		$p2 = "U";
+		$p3 = "U";
+
+		for ( $i = 4; $i < count( $seg ) - 3; ++$i ) {
+			$score = self::$model["BIAS"];
+			$w1 = $seg[$i - 3];
+			$w2 = $seg[$i - 2];
+			$w3 = $seg[$i - 1];
+			$w4 = $seg[$i];
+			$w5 = $seg[$i + 1];
+			$w6 = $seg[$i + 2];
+			$c1 = $ctype[$i - 3];
+			$c2 = $ctype[$i - 2];
+			$c3 = $ctype[$i - 1];
+			$c4 = $ctype[$i];
+			$c5 = $ctype[$i + 1];
+			$c6 = $ctype[$i + 2];
+			$score += $this->ts_( @self::$model["UP1"][$p1] );
+			$score += $this->ts_( @self::$model["UP2"][$p2] );
+			$score += $this->ts_( @self::$model["UP3"][$p3] );
+			$score += $this->ts_( @self::$model["BP1"][$p1 . $p2] );
+			$score += $this->ts_( @self::$model["BP2"][$p2 . $p3] );
+			$score += $this->ts_( @self::$model["UW1"][$w1] );
+			$score += $this->ts_( @self::$model["UW2"][$w2] );
+			$score += $this->ts_( @self::$model["UW3"][$w3] );
+			$score += $this->ts_( @self::$model["UW4"][$w4] );
+			$score += $this->ts_( @self::$model["UW5"][$w5] );
+			$score += $this->ts_( @self::$model["UW6"][$w6] );
+			$score += $this->ts_( @self::$model["BW1"][$w2 . $w3] );
+			$score += $this->ts_( @self::$model["BW2"][$w3 . $w4] );
+			$score += $this->ts_( @self::$model["BW3"][$w4 . $w5] );
+			$score += $this->ts_( @self::$model["TW1"][$w1 . $w2 . $w3] );
+			$score += $this->ts_( @self::$model["TW2"][$w2 . $w3 . $w4] );
+			$score += $this->ts_( @self::$model["TW3"][$w3 . $w4 . $w5] );
+			$score += $this->ts_( @self::$model["TW4"][$w4 . $w5 . $w6] );
+			$score += $this->ts_( @self::$model["UC1"][$c1] );
+			$score += $this->ts_( @self::$model["UC2"][$c2] );
+			$score += $this->ts_( @self::$model["UC3"][$c3] );
+			$score += $this->ts_( @self::$model["UC4"][$c4] );
+			$score += $this->ts_( @self::$model["UC5"][$c5] );
+			$score += $this->ts_( @self::$model["UC6"][$c6] );
+			$score += $this->ts_( @self::$model["BC1"][$c2 . $c3] );
+			$score += $this->ts_( @self::$model["BC2"][$c3 . $c4] );
+			$score += $this->ts_( @self::$model["BC3"][$c4 . $c5] );
+			$score += $this->ts_( @self::$model["TC1"][$c1 . $c2 . $c3] );
+			$score += $this->ts_( @self::$model["TC2"][$c2 . $c3 . $c4] );
+			$score += $this->ts_( @self::$model["TC3"][$c3 . $c4 . $c5] );
+			$score += $this->ts_( @self::$model["TC4"][$c4 . $c5 . $c6] );
+			// $score += $this->ts_(@self::$model["TC5"][$c4 . $c5 . $c6]);
+			$score += $this->ts_( @self::$model["UQ1"][$p1 . $c1] );
+			$score += $this->ts_( @self::$model["UQ2"][$p2 . $c2] );
+			$score += $this->ts_( @self::$model["UQ1"][$p3 . $c3] );
+			$score += $this->ts_( @self::$model["BQ1"][$p2 . $c2 . $c3] );
+			$score += $this->ts_( @self::$model["BQ2"][$p2 . $c3 . $c4] );
+			$score += $this->ts_( @self::$model["BQ3"][$p3 . $c2 . $c3] );
+			$score += $this->ts_( @self::$model["BQ4"][$p3 . $c3 . $c4] );
+			$score += $this->ts_( @self::$model["TQ1"][$p2 . $c1 . $c2 . $c3] );
+			$score += $this->ts_( @self::$model["TQ2"][$p2 . $c2 . $c3 . $c4] );
+			$score += $this->ts_( @self::$model["TQ3"][$p3 . $c1 . $c2 . $c3] );
+			$score += $this->ts_( @self::$model["TQ4"][$p3 . $c2 . $c3 . $c4] );
+
+			$p = "O";
+
+			if ( $score > 0 ) {
+
+				if ( $word !== '' && $word !== ' ' ) {
+					$result[] = $word;
+				}
+
+				$word = "";
+				$p = "B";
+			}
+
+			$p1 = $p2;
+			$p2 = $p3;
+			$p3 = $p;
+
+			if ( $seg[$i] !== '' && $seg[$i] !== ' ' ) {
+				$word .= $seg[$i];
+			}
+		}
+
+		$result[] = $word;
+
+		if ( $encoding !== 'UTF-8' ) {
+			foreach ( $result as &$str ) {
+				$str = mb_convert_encoding( $str, $encoding, 'UTF-8' );
+			}
+		}
+
+		return $result;
+	}
+
+	private function ctype_( $str ) {
+		foreach ( $this->patterns_ as $pattern => $type ) {
+			if ( preg_match( '/' . $pattern . '/u', $str ) ) {
+				return $type;
+			}
+		}
+
+		return "O";
+	}
+
+	private function ts_( $v ) {
+		return $v ? $v : 0;
+	}
+
+	private function mb_string_to_array_( $str, $encoding = 'UTF-8' ) {
+		$result = [];
+		$length = mb_strlen( $str, $encoding );
+
+		for ( $i = 0; $i < $length; ++$i ) {
+			$result[] = mb_substr( $str, $i, 1, $encoding );
+		}
+
+		return $result;
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/src/Tokenizer/NGramTokenizer.php

@@ -0,0 +1,127 @@
+<?php
+
+namespace Onoi\Tesa\Tokenizer;
+
+/**
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class NGramTokenizer implements Tokenizer {
+
+	/**
+	 * @var Tokenizer
+	 */
+	private $tokenizer;
+
+	/**
+	 * @var int
+	 */
+	private $ngramSize = 2;
+
+	/**
+	 * @var bool
+	 */
+	private $withMarker = false;
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param Tokenizer|null $tokenizer
+	 * @param int $ngramSize
+	 */
+	public function __construct( ?Tokenizer $tokenizer = null, $ngramSize = 2 ) {
+		$this->tokenizer = $tokenizer;
+		$this->ngramSize = (int)$ngramSize;
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param bool $withMarker
+	 */
+	public function withMarker( $withMarker ) {
+		$this->withMarker = (bool)$withMarker;
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param int $ngramSize
+	 */
+	public function setNgramSize( $ngramSize ) {
+		$this->ngramSize = (int)$ngramSize;
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * {@inheritDoc}
+	 */
+	public function setOption( $name, $value ) {
+		if ( $this->tokenizer !== null ) {
+			$this->tokenizer->setOption( $name, $value );
+		}
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * {@inheritDoc}
+	 */
+	public function isWordTokenizer() {
+		return false;
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * {@inheritDoc}
+	 */
+	public function tokenize( $string ) {
+		if ( $this->tokenizer !== null ) {
+			$string = implode( " ", $this->tokenizer->tokenize( $string ) );
+		}
+
+		$result = $this->createNGrams( $string, $this->ngramSize, $this->withMarker );
+
+		if ( $result !== false ) {
+			return $result;
+		}
+
+		return [];
+	}
+
+	private function createNGrams( $text, $ngramSize, $withMarker ) {
+		$ngramList = [];
+
+		// Identify the beginning-of-word and end-of-word
+		if ( $withMarker ) {
+			$text = '_' . str_replace( ' ', '_', $text );
+		}
+
+		$text = mb_strtolower( $text );
+		$textLength = mb_strlen( $text, 'UTF-8' );
+
+		for ( $i = 0; $i < $textLength; ++$i ) {
+
+			// Those failing the length of the ngramSize are skipped
+			if ( !$withMarker && $i + $ngramSize > $textLength ) {
+				continue;
+			}
+
+			$ngram = mb_substr( $text, $i, $ngramSize, 'UTF-8' );
+
+			// str_pad has issues with utf-8 length
+			if ( $withMarker && ( $ngl = mb_strlen( $ngram, 'UTF-8' ) ) < $ngramSize ) {
+				$ngram = $ngram . str_repeat( '_', ( $ngramSize - $ngl ) );
+			}
+
+			$ngramList[] = $ngram;
+		}
+
+		return array_values( $ngramList );
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/src/Tokenizer/PunctuationRegExTokenizer.php

@@ -0,0 +1,83 @@
+<?php
+
+namespace Onoi\Tesa\Tokenizer;
+
+/**
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class PunctuationRegExTokenizer implements Tokenizer {
+
+	/**
+	 * @var Tokenizer
+	 */
+	private $tokenizer;
+
+	/**
+	 * @var string
+	 */
+	private $patternExemption = '';
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param Tokenizer|null $tokenizer
+	 */
+	public function __construct( ?Tokenizer $tokenizer = null ) {
+		$this->tokenizer = $tokenizer;
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * {@inheritDoc}
+	 */
+	public function setOption( $name, $value ) {
+		if ( $this->tokenizer !== null ) {
+			$this->tokenizer->setOption( $name, $value );
+		}
+
+		if ( $name === self::REGEX_EXEMPTION ) {
+			$this->patternExemption = $value;
+		}
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * {@inheritDoc}
+	 */
+	public function isWordTokenizer() {
+		return $this->tokenizer !== null ? $this->tokenizer->isWordTokenizer() : true;
+	}
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param string $string
+	 *
+	 * @return array|false
+	 */
+	public function tokenize( $string ) {
+		if ( $this->tokenizer !== null ) {
+			$string = implode( " ", $this->tokenizer->tokenize( $string ) );
+		}
+
+		$pattern = str_replace(
+			$this->patternExemption,
+			'',
+			'＿－・，、；：！？．。…◆★◇□■（）【】《》〈〉；：“”＂〃＇｀［］｛｝｢｣＠＊＼／＆＃％｀＾＋＜＝＞｜～≪≫─＄＂_\-･,､;:!?.｡()[\]{}「」@*\/&#%`^+<=>|~«»$"\s'
+		);
+
+		$result = preg_split( '/[' . $pattern . ']+/u', $string, -1, PREG_SPLIT_NO_EMPTY );
+
+		if ( $result === false ) {
+			$result = [];
+		}
+
+		return $result;
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/src/Tokenizer/Tokenizer.php

@@ -0,0 +1,46 @@
+<?php
+
+namespace Onoi\Tesa\Tokenizer;
+
+/**
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+interface Tokenizer {
+
+	/**
+	 * Under some circumstances (used as search term) specific characters should
+	 * be exempted from the regular expression.
+	 */
+	const REGEX_EXEMPTION = 'regex.exemption';
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param string $name
+	 * @param mixed $value
+	 */
+	public function setOption( $name, $value );
+
+	/**
+	 * Some simple tokenizer may not rely on whitespaces to build
+	 * tokens.
+	 *
+	 * @since 0.1
+	 *
+	 * @return bool
+	 */
+	public function isWordTokenizer();
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param string $string
+	 *
+	 * @return array|false
+	 */
+	public function tokenize( $string );
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/src/Tokenizer/model/rwcp.model.json

@@ -0,0 +1,45 @@
+{
+	"BIAS":-332,
+	"BC1": {"HH":6,"II":2461,"KH":406,"OH":-1378},
+	"BC2": {"AA":-3267,"AI":2744,"AN":-878,"HH":-4070,"HM":-1711,"HN":4012,"HO":3761,"IA":1327,"IH":-1184,"II":-1332,"IK":1721,"IO":5492,"KI":3831,"KK":-8741,"MH":-3132,"MK":3334,"OO":-2920},
+	"BC3": {"HH":996,"HI":626,"HK":-721,"HN":-1307,"HO":-836,"IH":-301,"KK":2762,"MK":1079,"MM":4034,"OA":-1652,"OH":266},
+	"BP1": {"BB":295,"OB":304,"OO":-125,"UB":352},
+	"BP2": {"BO":60,"OO":-1762},
+	"BQ1": {"BHH":1150,"BHM":1521,"BII":-1158,"BIM":886,"BMH":1208,"BNH":449,"BOH":-91,"BOO":-2597,"OHI":451,"OIH":-296,"OKA":1851,"OKH":-1020,"OKK":904,"OOO":2965},
+	"BQ2": {"BHH":118,"BHI":-1159,"BHM":466,"BIH":-919,"BKK":-1720,"BKO":864,"OHH":-1139,"OHM":-181,"OIH":153,"UHI":-1146},
+	"BQ3": {"BHH":-792,"BHI":2664,"BII":-299,"BKI":419,"BMH":937,"BMM":8335,"BNN":998,"BOH":775,"OHH":2174,"OHM":439,"OII":280,"OKH":1798,"OKI":-793,"OKO":-2242,"OMH":-2402,"OOO":11699},
+	"BQ4": {"BHH":-3895,"BIH":3761,"BII":-4654,"BIK":1348,"BKK":-1806,"BMI":-3385,"BOO":-12396,"OAH":926,"OHH":266,"OHK":-2036,"ONN":-973},
+	"BW1": {",と":660,",同":727,"B1あ":1404,"B1同":542,"、と":660,"、同":727,"」と":1682,"あっ":1505,"いう":1743,"いっ":-2055,"いる":672,"うし":-4817,"うん":665,"から":3472,"がら":600,"こう":-790,"こと":2083,"こん":-1262,"さら":-4143,"さん":4573,"した":2641,"して":1104,"すで":-3399,"そこ":1977,"それ":-871,"たち":1122,"ため":601,"った":3463,"つい":-802,"てい":805,"てき":1249,"でき":1127,"です":3445,"では":844,"とい":-4915,"とみ":1922,"どこ":3887,"ない":5713,"なっ":3015,"など":7379,"なん":-1113,"にし":2468,"には":1498,"にも":1671,"に対":-912,"の一":-501,"の中":741,"ませ":2448,"まで":1711,"まま":2600,"まる":-2155,"やむ":-1947,"よっ":-2565,"れた":2369,"れで":-913,"をし":1860,"を見":731,"亡く":-1886,"京都":2558,"取り":-2784,"大き":-2604,"大阪":1497,"平方":-2314,"引き":-1336,"日本":-195,"本当":-2423,"毎日":-2113,"目指":-724,"Ｂ１あ":1404,"Ｂ１同":542,"｣と":1682},
+	"BW2": {"..":-11822,"11":-669,"――":-5730,"−−":-13175,"いう":-1609,"うか":2490,"かし":-1350,"かも":-602,"から":-7194,"かれ":4612,"がい":853,"がら":-3198,"きた":1941,"くな":-1597,"こと":-8392,"この":-4193,"させ":4533,"され":13168,"さん":-3977,"しい":-1819,"しか":-545,"した":5078,"して":972,"しな":939,"その":-3744,"たい":-1253,"たた":-662,"ただ":-3857,"たち":-786,"たと":1224,"たは":-939,"った":4589,"って":1647,"っと":-2094,"てい":6144,"てき":3640,"てく":2551,"ては":-3110,"ても":-3065,"でい":2666,"でき":-1528,"でし":-3828,"です":-4761,"でも":-4203,"とい":1890,"とこ":-1746,"とと":-2279,"との":720,"とみ":5168,"とも":-3941,"ない":-2488,"なが":-1313,"など":-6509,"なの":2614,"なん":3099,"にお":-1615,"にし":2748,"にな":2454,"によ":-7236,"に対":-14943,"に従":-4688,"に関":-11388,"のか":2093,"ので":-7059,"のに":-6041,"のの":-6125,"はい":1073,"はが":-1033,"はず":-2532,"ばれ":1813,"まし":-1316,"まで":-6621,"まれ":5409,"めて":-3153,"もい":2230,"もの":-10713,"らか":-944,"らし":-1611,"らに":-1897,"りし":651,"りま":1620,"れた":4270,"れて":849,"れば":4114,"ろう":6067,"われ":7901,"を通":-11877,"んだ":728,"んな":-4115,"一人":602,"一方":-1375,"一日":970,"一部":-1051,"上が":-4479,"会社":-1116,"出て":2163,"分の":-7758,"同党":970,"同日":-913,"大阪":-2471,"委員":-1250,"少な":-1050,"年度":-8669,"年間":-1626,"府県":-2363,"手権":-1982,"新聞":-4066,"日新":-722,"日本":-7068,"日米":3372,"曜日":-601,"朝鮮":-2355,"本人":-2697,"東京":-1543,"然と":-1384,"社会":-1276,"立て":-990,"第に":-1612,"米国":-4268,"１１":-669},
+	"BW3": {"あた":-2194,"あり":719,"ある":3846,"い.":-1185,"い。":-1185,"いい":5308,"いえ":2079,"いく":3029,"いた":2056,"いっ":1883,"いる":5600,"いわ":1527,"うち":1117,"うと":4798,"えと":1454,"か.":2857,"か。":2857,"かけ":-743,"かっ":-4098,"かに":-669,"から":6520,"かり":-2670,"が,":1816,"が、":1816,"がき":-4855,"がけ":-1127,"がっ":-913,"がら":-4977,"がり":-2064,"きた":1645,"けど":1374,"こと":7397,"この":1542,"ころ":-2757,"さい":-714,"さを":976,"し,":1557,"し、":1557,"しい":-3714,"した":3562,"して":1449,"しな":2608,"しま":1200,"す.":-1310,"す。":-1310,"する":6521,"ず,":3426,"ず、":3426,"ずに":841,"そう":428,"た.":8875,"た。":8875,"たい":-594,"たの":812,"たり":-1183,"たる":-853,"だ.":4098,"だ。":4098,"だっ":1004,"った":-4748,"って":300,"てい":6240,"てお":855,"ても":302,"です":1437,"でに":-1482,"では":2295,"とう":-1387,"とし":2266,"との":541,"とも":-3543,"どう":4664,"ない":1796,"なく":-903,"など":2135,"に,":-1021,"に、":-1021,"にし":1771,"にな":1906,"には":2644,"の,":-724,"の、":-724,"の子":-1000,"は,":1337,"は、":1337,"べき":2181,"まし":1113,"ます":6943,"まっ":-1549,"まで":6154,"まれ":-793,"らし":1479,"られ":6820,"るる":3818,"れ,":854,"れ、":854,"れた":1850,"れて":1375,"れば":-3246,"れる":1091,"われ":-605,"んだ":606,"んで":798,"カ月":990,"会議":860,"入り":1232,"大会":2217,"始め":1681,"市":965,"新聞":-5055,"日,":974,"日、":974,"社会":2024,"ｶ月":990},
+	"TC1": {"AAA":1093,"HHH":1029,"HHM":580,"HII":998,"HOH":-390,"HOM":-331,"IHI":1169,"IOH":-142,"IOI":-1015,"IOM":467,"MMH":187,"OOI":-1832},
+	"TC2": {"HHO":2088,"HII":-1023,"HMM":-1154,"IHI":-1965,"KKH":703,"OII":-2649},
+	"TC3": {"AAA":-294,"HHH":346,"HHI":-341,"HII":-1088,"HIK":731,"HOH":-1486,"IHH":128,"IHI":-3041,"IHO":-1935,"IIH":-825,"IIM":-1035,"IOI":-542,"KHH":-1216,"KKA":491,"KKH":-1217,"KOK":-1009,"MHH":-2694,"MHM":-457,"MHO":123,"MMH":-471,"NNH":-1689,"NNO":662,"OHO":-3393},
+	"TC4": {"HHH":-203,"HHI":1344,"HHK":365,"HHM":-122,"HHN":182,"HHO":669,"HIH":804,"HII":679,"HOH":446,"IHH":695,"IHO":-2324,"IIH":321,"III":1497,"IIO":656,"IOO":54,"KAK":4845,"KKA":3386,"KKK":3065,"MHH":-405,"MHI":201,"MMH":-241,"MMM":661,"MOM":841},
+	"TQ1": {"BHHH":-227,"BHHI":316,"BHIH":-132,"BIHH":60,"BIII":1595,"BNHH":-744,"BOHH":225,"BOOO":-908,"OAKK":482,"OHHH":281,"OHIH":249,"OIHI":200,"OIIH":-68},
+	"TQ2": {"BIHH":-1401,"BIII":-1033,"BKAK":-543,"BOOO":-5591},
+	"TQ3": {"BHHH":478,"BHHM":-1073,"BHIH":222,"BHII":-504,"BIIH":-116,"BIII":-105,"BMHI":-863,"BMHM":-464,"BOMH":620,"OHHH":346,"OHHI":1729,"OHII":997,"OHMH":481,"OIHH":623,"OIIH":1344,"OKAK":2792,"OKHH":587,"OKKA":679,"OOHH":110,"OOII":-685},
+	"TQ4": {"BHHH":-721,"BHHM":-3604,"BHII":-966,"BIIH":-607,"BIII":-2181,"OAAA":-2763,"OAKK":180,"OHHH":-294,"OHHI":2446,"OHHO":480,"OHIH":-1573,"OIHH":1935,"OIHI":-493,"OIIH":626,"OIII":-4007,"OKAK":-8156},
+	"TW1": {"につい":-4681,"東京都":2026},
+	"TW2": {"ある程":-2049,"いった":-1256,"ころが":-2434,"しょう":3873,"その後":-4430,"だって":-1049,"ていた":1833,"として":-4657,"ともに":-4517,"もので":1882,"一気に":-792,"初めて":-1512,"同時に":-8097,"大きな":-1255,"対して":-2721,"社会党":-3216},
+	"TW3": {"いただ":-1734,"してい":1314,"として":-4314,"につい":-5483,"にとっ":-5989,"に当た":-6247,"ので,":-727,"ので、":-727,"のもの":-600,"れから":-3752,"十二月":-2287},
+	"TW4": {"いう.":8576,"いう。":8576,"からな":-2348,"してい":2958,"たが,":1516,"たが、":1516,"ている":1538,"という":1349,"ました":5543,"ません":1097,"ようと":-4258,"よると":5865},
+	"UC1": {"A":484,"K":93,"M":645,"O":-505},
+	"UC2": {"A":819,"H":1059,"I":409,"M":3987,"N":5775,"O":646},
+	"UC3": {"A":-1370,"I":2311},
+	"UC4": {"A":-2643,"H":1809,"I":-1032,"K":-3450,"M":3565,"N":3876,"O":6646},
+	"UC5": {"H":313,"I":-1238,"K":-799,"M":539,"O":-831},
+	"UC6": {"H":-506,"I":-253,"K":87,"M":247,"O":-387},
+	"UP1": {"O":-214},
+	"UP2": {"B":69,"O":935},
+	"UP3": {"B":189},
+	"UQ1": {"BH":21,"BI":-12,"BK":-99,"BN":142,"BO":-56,"OH":-95,"OI":477,"OK":410,"OO":-2422},
+	"UQ2": {"BH":216,"BI":113,"OK":1759},
+	"UQ3": {"BA":-479,"BH":42,"BI":1913,"BK":-7198,"BM":3160,"BN":6427,"BO":14761,"OI":-827,"ON":-3212},
+	"UW1": {",":156,"、":156,"「":-463,"あ":-941,"う":-127,"が":-553,"き":121,"こ":505,"で":-201,"と":-547,"ど":-123,"に":-789,"の":-185,"は":-847,"も":-466,"や":-470,"よ":182,"ら":-292,"り":208,"れ":169,"を":-446,"ん":-137,"・":-135,"主":-402,"京":-268,"区":-912,"午":871,"国":-460,"大":561,"委":729,"市":-411,"日":-141,"理":361,"生":-408,"県":-386,"都":-718,"｢":-463,"･":-135},
+	"UW2": {",":-829,"、":-829,"〇":892,"「":-645,"」":3145,"あ":-538,"い":505,"う":134,"お":-502,"か":1454,"が":-856,"く":-412,"こ":1141,"さ":878,"ざ":540,"し":1529,"す":-675,"せ":300,"そ":-1011,"た":188,"だ":1837,"つ":-949,"て":-291,"で":-268,"と":-981,"ど":1273,"な":1063,"に":-1764,"の":130,"は":-409,"ひ":-1273,"べ":1261,"ま":600,"も":-1263,"や":-402,"よ":1639,"り":-579,"る":-694,"れ":571,"を":-2516,"ん":2095,"ア":-587,"カ":306,"キ":568,"ッ":831,"三":-758,"不":-2150,"世":-302,"中":-968,"主":-861,"事":492,"人":-123,"会":978,"保":362,"入":548,"初":-3025,"副":-1566,"北":-3414,"区":-422,"大":-1769,"天":-865,"太":-483,"子":-1519,"学":760,"実":1023,"小":-2009,"市":-813,"年":-1060,"強":1067,"手":-1519,"揺":-1033,"政":1522,"文":-1355,"新":-1682,"日":-1815,"明":-1462,"最":-630,"朝":-1843,"本":-1650,"東":-931,"果":-665,"次":-2378,"民":-180,"気":-1740,"理":752,"発":529,"目":-1584,"相":-242,"県":-1165,"立":-763,"第":810,"米":509,"自":-1353,"行":838,"西":-744,"見":-3874,"調":1010,"議":1198,"込":3041,"開":1758,"間":-1257,"｢":-645,"｣":3145,"ｯ":831,"ｱ":-587,"ｶ":306,"ｷ":568},
+	"UW3": {",":4889,"1":-800,"−":-1723,"、":4889,"々":-2311,"〇":5827,"」":2670,"〓":-3573,"あ":-2696,"い":1006,"う":2342,"え":1983,"お":-4864,"か":-1163,"が":3271,"く":1004,"け":388,"げ":401,"こ":-3552,"ご":-3116,"さ":-1058,"し":-395,"す":584,"せ":3685,"そ":-5228,"た":842,"ち":-521,"っ":-1444,"つ":-1081,"て":6167,"で":2318,"と":1691,"ど":-899,"な":-2788,"に":2745,"の":4056,"は":4555,"ひ":-2171,"ふ":-1798,"へ":1199,"ほ":-5516,"ま":-4384,"み":-120,"め":1205,"も":2323,"や":-788,"よ":-202,"ら":727,"り":649,"る":5905,"れ":2773,"わ":-1207,"を":6620,"ん":-518,"ア":551,"グ":1319,"ス":874,"ッ":-1350,"ト":521,"ム":1109,"ル":1591,"ロ":2201,"ン":278,"・":-3794,"一":-1619,"下":-1759,"世":-2087,"両":3815,"中":653,"主":-758,"予":-1193,"二":974,"人":2742,"今":792,"他":1889,"以":-1368,"低":811,"何":4265,"作":-361,"保":-2439,"元":4858,"党":3593,"全":1574,"公":-3030,"六":755,"共":-1880,"円":5807,"再":3095,"分":457,"初":2475,"別":1129,"前":2286,"副":4437,"力":365,"動":-949,"務":-1872,"化":1327,"北":-1038,"区":4646,"千":-2309,"午":-783,"協":-1006,"口":483,"右":1233,"各":3588,"合":-241,"同":3906,"和":-837,"員":4513,"国":642,"型":1389,"場":1219,"外":-241,"妻":2016,"学":-1356,"安":-423,"実":-1008,"家":1078,"小":-513,"少":-3102,"州":1155,"市":3197,"平":-1804,"年":2416,"広":-1030,"府":1605,"度":1452,"建":-2352,"当":-3885,"得":1905,"思":-1291,"性":1822,"戸":-488,"指":-3973,"政":-2013,"教":-1479,"数":3222,"文":-1489,"新":1764,"日":2099,"旧":5792,"昨":-661,"時":-1248,"曜":-951,"最":-937,"月":4125,"期":360,"李":3094,"村":364,"東":-805,"核":5156,"森":2438,"業":484,"氏":2613,"民":-1694,"決":-1073,"法":1868,"海":-495,"無":979,"物":461,"特":-3850,"生":-273,"用":914,"町":1215,"的":7313,"直":-1835,"省":792,"県":6293,"知":-1528,"私":4231,"税":401,"立":-960,"第":1201,"米":7767,"系":3066,"約":3663,"級":1384,"統":-4229,"総":1163,"線":1255,"者":6457,"能":725,"自":-2869,"英":785,"見":1044,"調":-562,"財":-733,"費":1777,"車":1835,"軍":1375,"込":-1504,"通":-1136,"選":-681,"郎":1026,"郡":4404,"部":1200,"金":2163,"長":421,"開":-1432,"間":1302,"関":-1282,"雨":2009,"電":-1045,"非":2066,"駅":1620,"１":-800,"｣":2670,"･":-3794,"ｯ":-1350,"ｱ":551,"ｸﾞ":1319,"ｽ":874,"ﾄ":521,"ﾑ":1109,"ﾙ":1591,"ﾛ":2201,"ﾝ":278},
+	"UW4": {",":3930,".":3508,"―":-4841,"、":3930,"。":3508,"〇":4999,"「":1895,"」":3798,"〓":-5156,"あ":4752,"い":-3435,"う":-640,"え":-2514,"お":2405,"か":530,"が":6006,"き":-4482,"ぎ":-3821,"く":-3788,"け":-4376,"げ":-4734,"こ":2255,"ご":1979,"さ":2864,"し":-843,"じ":-2506,"す":-731,"ず":1251,"せ":181,"そ":4091,"た":5034,"だ":5408,"ち":-3654,"っ":-5882,"つ":-1659,"て":3994,"で":7410,"と":4547,"な":5433,"に":6499,"ぬ":1853,"ね":1413,"の":7396,"は":8578,"ば":1940,"ひ":4249,"び":-4134,"ふ":1345,"へ":6665,"べ":-744,"ほ":1464,"ま":1051,"み":-2082,"む":-882,"め":-5046,"も":4169,"ゃ":-2666,"や":2795,"ょ":-1544,"よ":3351,"ら":-2922,"り":-9726,"る":-14896,"れ":-2613,"ろ":-4570,"わ":-1783,"を":13150,"ん":-2352,"カ":2145,"コ":1789,"セ":1287,"ッ":-724,"ト":-403,"メ":-1635,"ラ":-881,"リ":-541,"ル":-856,"ン":-3637,"・":-4371,"ー":-11870,"一":-2069,"中":2210,"予":782,"事":-190,"井":-1768,"人":1036,"以":544,"会":950,"体":-1286,"作":530,"側":4292,"先":601,"党":-2006,"共":-1212,"内":584,"円":788,"初":1347,"前":1623,"副":3879,"力":-302,"動":-740,"務":-2715,"化":776,"区":4517,"協":1013,"参":1555,"合":-1834,"和":-681,"員":-910,"器":-851,"回":1500,"国":-619,"園":-1200,"地":866,"場":-1410,"塁":-2094,"士":-1413,"多":1067,"大":571,"子":-4802,"学":-1397,"定":-1057,"寺":-809,"小":1910,"屋":-1328,"山":-1500,"島":-2056,"川":-2667,"市":2771,"年":374,"庁":-4556,"後":456,"性":553,"感":916,"所":-1566,"支":856,"改":787,"政":2182,"教":704,"文":522,"方":-856,"日":1798,"時":1829,"最":845,"月":-9066,"木":-485,"来":-442,"校":-360,"業":-1043,"氏":5388,"民":-2716,"気":-910,"沢":-939,"済":-543,"物":-735,"率":672,"球":-1267,"生":-1286,"産":-1101,"田":-2900,"町":1826,"的":2586,"目":922,"省":-3485,"県":2997,"空":-867,"立":-2112,"第":788,"米":2937,"系":786,"約":2171,"経":1146,"統":-1169,"総":940,"線":-994,"署":749,"者":2145,"能":-730,"般":-852,"行":-792,"規":792,"警":-1184,"議":-244,"谷":-1000,"賞":730,"車":-1481,"軍":1158,"輪":-1433,"込":-3370,"近":929,"道":-1291,"選":2596,"郎":-4866,"都":1192,"野":-1100,"銀":-2213,"長":357,"間":-2344,"院":-2297,"際":-2604,"電":-878,"領":-1659,"題":-792,"館":-1984,"首":1749,"高":2120,"｢":1895,"｣":3798,"･":-4371,"ｯ":-724,"ｰ":-11870,"ｶ":2145,"ｺ":1789,"ｾ":1287,"ﾄ":-403,"ﾒ":-1635,"ﾗ":-881,"ﾘ":-541,"ﾙ":-856,"ﾝ":-3637},
+	"UW5": {",":465,".":-299,"1":-514,"E2":-32768,"]":-2762,"、":465,"。":-299,"「":363,"あ":1655,"い":331,"う":-503,"え":1199,"お":527,"か":647,"が":-421,"き":1624,"ぎ":1971,"く":312,"げ":-983,"さ":-1537,"し":-1371,"す":-852,"だ":-1186,"ち":1093,"っ":52,"つ":921,"て":-18,"で":-850,"と":-127,"ど":1682,"な":-787,"に":-1224,"の":-635,"は":-578,"べ":1001,"み":502,"め":865,"ゃ":3350,"ょ":854,"り":-208,"る":429,"れ":504,"わ":419,"を":-1264,"ん":327,"イ":241,"ル":451,"ン":-343,"中":-871,"京":722,"会":-1153,"党":-654,"務":3519,"区":-901,"告":848,"員":2104,"大":-1296,"学":-548,"定":1785,"嵐":-1304,"市":-2991,"席":921,"年":1763,"思":872,"所":-814,"挙":1618,"新":-1682,"日":218,"月":-4353,"査":932,"格":1356,"機":-1508,"氏":-1347,"田":240,"町":-3912,"的":-3149,"相":1319,"省":-1052,"県":-4003,"研":-997,"社":-278,"空":-813,"統":1955,"者":-2233,"表":663,"語":-1073,"議":1219,"選":-1018,"郎":-368,"長":786,"間":1191,"題":2368,"館":-689,"１":-514,"Ｅ２":-32768,"｢":363,"ｲ":241,"ﾙ":451,"ﾝ":-343},
+	"UW6": {",":227,".":808,"1":-270,"E1":306,"、":227,"。":808,"あ":-307,"う":189,"か":241,"が":-73,"く":-121,"こ":-200,"じ":1782,"す":383,"た":-428,"っ":573,"て":-1014,"で":101,"と":-105,"な":-253,"に":-149,"の":-417,"は":-236,"も":-206,"り":187,"る":-135,"を":195,"ル":-673,"ン":-496,"一":-277,"中":201,"件":-800,"会":624,"前":302,"区":1792,"員":-1212,"委":798,"学":-960,"市":887,"広":-695,"後":535,"業":-697,"相":753,"社":-507,"福":974,"空":-822,"者":1811,"連":463,"郎":1082,"１":-270,"Ｅ１":306,"ﾙ":-673,"ﾝ":-496}
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/src/Transliterator.php

@@ -0,0 +1,303 @@
+<?php
+
+namespace Onoi\Tesa;
+
+/**
+ * Romanization and transliteration support for a selected group of characters.
+ *
+ * @see http://www.loc.gov/catdir/cpso/roman.html
+ * @see https://en.wikipedia.org/wiki/List_of_ISO_transliterations
+ *
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class Transliterator {
+
+	/**
+	 * Any change to the mapping of this file should be reflected in a version
+	 * change (the version number does not necessarily correlate with the
+	 * library version)
+	 */
+	const VERSION = '0.2';
+
+	/**
+	 * Transliterator supported option flags
+	 */
+	const NONE = 0x1;
+	const DIACRITICS = 0x2;
+	const GREEK = 0x4;
+
+	/**
+	 * Convert diacritics (e.g. accents) of a string to a Latin representation
+	 *
+	 * @see http://jsperf.com/latinize
+	 *
+	 * - Changed the German conversion rules for the diaeresis representation
+	 *   (Ü to Ue, Ä to ae etc.)
+	 * - Added ß/ss
+	 *
+	 *
+	 * @var array
+	 */
+	private static $diacriticsMap = [
+			"Á" => "A", "Ă" => "A", "Ắ" => "A", "Ặ" => "A", "Ằ" => "A", "Ẳ" => "A", "Ẵ" => "A",
+			"Ǎ" => "A", "Â" => "A", "Ấ" => "A", "Ậ" => "A", "Ầ" => "A", "Ẩ" => "A", "Ẫ" => "A",
+			"Ä" => "AE", "Ǟ" => "A", "Ȧ" => "A", "Ǡ" => "A", "Ạ" => "A", "Ȁ" => "A", "À" => "A",
+			"Ả" => "A", "Ȃ" => "A", "Ā" => "A", "Ą" => "A", "Å" => "A", "Ǻ" => "A", "Ḁ" => "A",
+			"Ⱥ" => "A", "Ã" => "A", "Ꜳ" => "AA", "Æ" => "AE", "Ǽ" => "AE", "Ǣ" => "AE", "Ꜵ" => "AO",
+			"Ꜷ" => "AU", "Ꜹ" => "AV", "Ꜻ" => "AV", "Ꜽ" => "AY", "Ḃ" => "B", "Ḅ" => "B", "Ɓ" => "B",
+			"Ḇ" => "B", "Ƀ" => "B", "Ƃ" => "B", "Ć" => "C", "Č" => "C", "Ç" => "C", "Ḉ" => "C",
+			"Ĉ" => "C", "Ċ" => "C", "Ƈ" => "C", "Ȼ" => "C", "Ď" => "D", "Ḑ" => "D", "Ḓ" => "D",
+			"Ḋ" => "D", "Ḍ" => "D", "Ɗ" => "D", "Ḏ" => "D", "Dz" => "D", "Dž" => "D", "Đ" => "D",
+			"Ƌ" => "D", "DZ" => "DZ", "DŽ" => "DZ", "É" => "E", "Ĕ" => "E", "Ě" => "E", "Ȩ" => "E",
+			"Ḝ" => "E", "Ê" => "E", "Ế" => "E", "Ệ" => "E", "Ề" => "E", "Ể" => "E", "Ễ" => "E",
+			"Ḙ" => "E", "Ë" => "E", "Ė" => "E", "Ẹ" => "E", "Ȅ" => "E", "È" => "E", "Ẻ" => "E",
+			"Ȇ" => "E", "Ē" => "E", "Ḗ" => "E", "Ḕ" => "E", "Ę" => "E", "Ɇ" => "E", "Ẽ" => "E",
+			"Ḛ" => "E", "Ꝫ" => "ET", "Ḟ" => "F", "Ƒ" => "F", "Ǵ" => "G", "Ğ" => "G", "Ǧ" => "G",
+			"Ģ" => "G", "Ĝ" => "G", "Ġ" => "G", "Ɠ" => "G", "Ḡ" => "G", "Ǥ" => "G", "Ḫ" => "H",
+			"Ȟ" => "H", "Ḩ" => "H", "Ĥ" => "H", "Ⱨ" => "H", "Ḧ" => "H", "Ḣ" => "H", "Ḥ" => "H",
+			"Ħ" => "H", "Í" => "I", "Ĭ" => "I", "Ǐ" => "I", "Î" => "I", "Ï" => "I", "Ḯ" => "I",
+			"İ" => "I", "Ị" => "I", "Ȉ" => "I", "Ì" => "I", "Ỉ" => "I", "Ȋ" => "I", "Ī" => "I",
+			"Į" => "I", "Ɨ" => "I", "Ĩ" => "I", "Ḭ" => "I", "Ꝺ" => "D", "Ꝼ" => "F", "Ᵹ" => "G",
+			"Ꞃ" => "R", "Ꞅ" => "S", "Ꞇ" => "T", "Ꝭ" => "IS", "Ĵ" => "J", "Ɉ" => "J", "Ḱ" => "K",
+			"Ǩ" => "K", "Ķ" => "K", "Ⱪ" => "K", "Ꝃ" => "K", "Ḳ" => "K", "Ƙ" => "K", "Ḵ" => "K",
+			"Ꝁ" => "K", "Ꝅ" => "K", "Ĺ" => "L", "Ƚ" => "L", "Ľ" => "L", "Ļ" => "L", "Ḽ" => "L",
+			"Ḷ" => "L", "Ḹ" => "L", "Ⱡ" => "L", "Ꝉ" => "L", "Ḻ" => "L", "Ŀ" => "L", "Ɫ" => "L",
+			"Lj" => "L", "Ł" => "L", "LJ" => "LJ", "Ḿ" => "M", "Ṁ" => "M", "Ṃ" => "M", "Ɱ" => "M",
+			"Ń" => "N", "Ň" => "N", "Ņ" => "N", "Ṋ" => "N", "Ṅ" => "N", "Ṇ" => "N", "Ǹ" => "N",
+			"Ɲ" => "N", "Ṉ" => "N", "Ƞ" => "N", "Nj" => "N", "Ñ" => "N", "NJ" => "NJ", "Ó" => "O",
+			"Ŏ" => "O", "Ǒ" => "O", "Ô" => "O", "Ố" => "O", "Ộ" => "O", "Ồ" => "O", "Ổ" => "O",
+			"Ỗ" => "O", "Ö" => "OE", "Ȫ" => "O", "Ȯ" => "O", "Ȱ" => "O", "Ọ" => "O", "Ő" => "O",
+			"Ȍ" => "O", "Ò" => "O", "Ỏ" => "O", "Ơ" => "O", "Ớ" => "O", "Ợ" => "O", "Ờ" => "O",
+			"Ở" => "O", "Ỡ" => "O", "Ȏ" => "O", "Ꝋ" => "O", "Ꝍ" => "O", "Ō" => "O", "Ṓ" => "O",
+			"Ṑ" => "O", "Ɵ" => "O", "Ǫ" => "O", "Ǭ" => "O", "Ø" => "O", "Ǿ" => "O", "Õ" => "O",
+			"Ṍ" => "O", "Ṏ" => "O", "Ȭ" => "O", "Ƣ" => "OI", "Ꝏ" => "OO", "Ɛ" => "E", "Ɔ" => "O",
+			"Ȣ" => "OU", "Ṕ" => "P", "Ṗ" => "P", "Ꝓ" => "P", "Ƥ" => "P", "Ꝕ" => "P", "Ᵽ" => "P",
+			"Ꝑ" => "P", "Ꝙ" => "Q", "Ꝗ" => "Q", "Ŕ" => "R", "Ř" => "R", "Ŗ" => "R", "Ṙ" => "R",
+			"Ṛ" => "R", "Ṝ" => "R", "Ȑ" => "R", "Ȓ" => "R", "Ṟ" => "R", "Ɍ" => "R", "Ɽ" => "R",
+			"Ꜿ" => "C", "Ǝ" => "E", "Ś" => "S", "Ṥ" => "S", "Š" => "S", "Ṧ" => "S", "Ş" => "S",
+			"Ŝ" => "S", "Ș" => "S", "Ṡ" => "S", "Ṣ" => "S", "Ṩ" => "S", "ß" => "ss",
+			"Ť" => "T", "Ţ" => "T",
+			"Ṱ" => "T", "Ț" => "T", "Ⱦ" => "T", "Ṫ" => "T", "Ṭ" => "T", "Ƭ" => "T", "Ṯ" => "T",
+			"Ʈ" => "T", "Ŧ" => "T", "Ɐ" => "A", "Ꞁ" => "L", "Ɯ" => "M", "Ʌ" => "V", "Ꜩ" => "TZ",
+			"Ú" => "U", "Ŭ" => "U", "Ǔ" => "U", "Û" => "U", "Ṷ" => "U", "Ü" => "UE", "Ǘ" => "U",
+			"Ǚ" => "U", "Ǜ" => "U", "Ǖ" => "U", "Ṳ" => "U", "Ụ" => "U", "Ű" => "U", "Ȕ" => "U",
+			"Ù" => "U", "Ủ" => "U", "Ư" => "U", "Ứ" => "U", "Ự" => "U", "Ừ" => "U", "Ử" => "U",
+			"Ữ" => "U", "Ȗ" => "U", "Ū" => "U", "Ṻ" => "U", "Ų" => "U", "Ů" => "U", "Ũ" => "U",
+			"Ṹ" => "U", "Ṵ" => "U", "Ꝟ" => "V", "Ṿ" => "V", "Ʋ" => "V", "Ṽ" => "V", "Ꝡ" => "VY",
+			"Ẃ" => "W", "Ŵ" => "W", "Ẅ" => "W", "Ẇ" => "W", "Ẉ" => "W", "Ẁ" => "W", "Ⱳ" => "W",
+			"Ẍ" => "X", "Ẋ" => "X", "Ý" => "Y", "Ŷ" => "Y", "Ÿ" => "Y", "Ẏ" => "Y", "Ỵ" => "Y",
+			"Ỳ" => "Y", "Ƴ" => "Y", "Ỷ" => "Y", "Ỿ" => "Y", "Ȳ" => "Y", "Ɏ" => "Y", "Ỹ" => "Y",
+			"Ź" => "Z", "Ž" => "Z", "Ẑ" => "Z", "Ⱬ" => "Z", "Ż" => "Z", "Ẓ" => "Z", "Ȥ" => "Z",
+			"Ẕ" => "Z", "Ƶ" => "Z", "IJ" => "IJ", "Œ" => "OE", "ᴀ" => "A", "ᴁ" => "AE", "ʙ" => "B",
+			"ᴃ" => "B", "ᴄ" => "C", "ᴅ" => "D", "ᴇ" => "E", "ꜰ" => "F", "ɢ" => "G", "ʛ" => "G",
+			"ʜ" => "H", "ɪ" => "I", "ʁ" => "R", "ᴊ" => "J", "ᴋ" => "K", "ʟ" => "L", "ᴌ" => "L",
+			"ᴍ" => "M", "ɴ" => "N", "ᴏ" => "O", "ɶ" => "OE", "ᴐ" => "O", "ᴕ" => "OU", "ᴘ" => "P",
+			"ʀ" => "R", "ᴎ" => "N", "ᴙ" => "R", "ꜱ" => "S", "ᴛ" => "T", "ⱻ" => "E", "ᴚ" => "R",
+			"ᴜ" => "U", "ᴠ" => "V", "ᴡ" => "W", "ʏ" => "Y", "ᴢ" => "Z", "á" => "a", "ă" => "a",
+			"ắ" => "a", "ặ" => "a", "ằ" => "a", "ẳ" => "a", "ẵ" => "a", "ǎ" => "a", "â" => "a",
+			"ấ" => "a", "ậ" => "a", "ầ" => "a", "ẩ" => "a", "ẫ" => "a", "ä" => "ae", "ǟ" => "a",
+			"ȧ" => "a", "ǡ" => "a", "ạ" => "a", "ȁ" => "a", "à" => "a", "ả" => "a", "ȃ" => "a",
+			"ā" => "a", "ą" => "a", "ᶏ" => "a", "ẚ" => "a", "å" => "a", "ǻ" => "a", "ḁ" => "a",
+			"ⱥ" => "a", "ã" => "a", "ꜳ" => "aa", "æ" => "ae", "ǽ" => "ae", "ǣ" => "ae", "ꜵ" => "ao",
+			"ꜷ" => "au", "ꜹ" => "av", "ꜻ" => "av", "ꜽ" => "ay", "ḃ" => "b", "ḅ" => "b", "ɓ" => "b",
+			"ḇ" => "b", "ᵬ" => "b", "ᶀ" => "b", "ƀ" => "b", "ƃ" => "b", "ɵ" => "o", "ć" => "c",
+			"č" => "c", "ç" => "c", "ḉ" => "c", "ĉ" => "c", "ɕ" => "c", "ċ" => "c", "ƈ" => "c",
+			"ȼ" => "c", "ď" => "d", "ḑ" => "d", "ḓ" => "d", "ȡ" => "d", "ḋ" => "d", "ḍ" => "d",
+			"ɗ" => "d", "ᶑ" => "d", "ḏ" => "d", "ᵭ" => "d", "ᶁ" => "d", "đ" => "d", "ɖ" => "d",
+			"ƌ" => "d", "ı" => "i", "ȷ" => "j", "ɟ" => "j", "ʄ" => "j", "dz" => "dz", "dž" => "dz",
+			"é" => "e", "ĕ" => "e", "ě" => "e", "ȩ" => "e", "ḝ" => "e", "ê" => "e", "ế" => "e",
+			"ệ" => "e", "ề" => "e", "ể" => "e", "ễ" => "e", "ḙ" => "e", "ë" => "e", "ė" => "e",
+			"ẹ" => "e", "ȅ" => "e", "è" => "e", "ẻ" => "e", "ȇ" => "e", "ē" => "e", "ḗ" => "e",
+			"ḕ" => "e", "ⱸ" => "e", "ę" => "e", "ᶒ" => "e", "ɇ" => "e", "ẽ" => "e", "ḛ" => "e",
+			"ꝫ" => "et", "ḟ" => "f", "ƒ" => "f", "ᵮ" => "f", "ᶂ" => "f", "ǵ" => "g", "ğ" => "g",
+			"ǧ" => "g", "ģ" => "g", "ĝ" => "g", "ġ" => "g", "ɠ" => "g", "ḡ" => "g", "ᶃ" => "g",
+			"ǥ" => "g", "ḫ" => "h", "ȟ" => "h", "ḩ" => "h", "ĥ" => "h", "ⱨ" => "h", "ḧ" => "h",
+			"ḣ" => "h", "ḥ" => "h", "ɦ" => "h", "ẖ" => "h", "ħ" => "h", "ƕ" => "hv", "í" => "i",
+			"ĭ" => "i", "ǐ" => "i", "î" => "i", "ï" => "i", "ḯ" => "i", "ị" => "i", "ȉ" => "i",
+			"ì" => "i", "ỉ" => "i", "ȋ" => "i", "ī" => "i", "į" => "i", "ᶖ" => "i", "ɨ" => "i",
+			"ĩ" => "i", "ḭ" => "i", "ꝺ" => "d", "ꝼ" => "f", "ᵹ" => "g", "ꞃ" => "r", "ꞅ" => "s",
+			"ꞇ" => "t", "ꝭ" => "is", "ǰ" => "j", "ĵ" => "j", "ʝ" => "j", "ɉ" => "j", "ḱ" => "k",
+			"ǩ" => "k", "ķ" => "k", "ⱪ" => "k", "ꝃ" => "k", "ḳ" => "k", "ƙ" => "k", "ḵ" => "k",
+			"ᶄ" => "k", "ꝁ" => "k", "ꝅ" => "k", "ĺ" => "l", "ƚ" => "l", "ɬ" => "l", "ľ" => "l",
+			"ļ" => "l", "ḽ" => "l", "ȴ" => "l", "ḷ" => "l", "ḹ" => "l", "ⱡ" => "l", "ꝉ" => "l",
+			"ḻ" => "l", "ŀ" => "l", "ɫ" => "l", "ᶅ" => "l", "ɭ" => "l", "ł" => "l", "lj" => "lj",
+			"ſ" => "s", "ẜ" => "s", "ẛ" => "s", "ẝ" => "s", "ḿ" => "m", "ṁ" => "m", "ṃ" => "m",
+			"ɱ" => "m", "ᵯ" => "m", "ᶆ" => "m", "ń" => "n", "ň" => "n", "ņ" => "n", "ṋ" => "n",
+			"ȵ" => "n", "ṅ" => "n", "ṇ" => "n", "ǹ" => "n", "ɲ" => "n", "ṉ" => "n", "ƞ" => "n",
+			"ᵰ" => "n", "ᶇ" => "n", "ɳ" => "n", "ñ" => "n", "nj" => "nj", "ó" => "o", "ŏ" => "o",
+			"ǒ" => "o", "ô" => "o", "ố" => "o", "ộ" => "o", "ồ" => "o", "ổ" => "o", "ỗ" => "o",
+			"ö" => "oe", "ȫ" => "o", "ȯ" => "o", "ȱ" => "o", "ọ" => "o", "ő" => "o", "ȍ" => "o",
+			"ò" => "o", "ỏ" => "o", "ơ" => "o", "ớ" => "o", "ợ" => "o", "ờ" => "o", "ở" => "o",
+			"ỡ" => "o", "ȏ" => "o", "ꝋ" => "o", "ꝍ" => "o", "ⱺ" => "o", "ō" => "o", "ṓ" => "o",
+			"ṑ" => "o", "ǫ" => "o", "ǭ" => "o", "ø" => "o", "ǿ" => "o", "õ" => "o", "ṍ" => "o",
+			"ṏ" => "o", "ȭ" => "o", "ƣ" => "oi", "ꝏ" => "oo", "ɛ" => "e", "ᶓ" => "e", "ɔ" => "o",
+			"ᶗ" => "o", "ȣ" => "ou", "ṕ" => "p", "ṗ" => "p", "ꝓ" => "p", "ƥ" => "p", "ᵱ" => "p",
+			"ᶈ" => "p", "ꝕ" => "p", "ᵽ" => "p", "ꝑ" => "p", "ꝙ" => "q", "ʠ" => "q", "ɋ" => "q",
+			"ꝗ" => "q", "ŕ" => "r", "ř" => "r", "ŗ" => "r", "ṙ" => "r", "ṛ" => "r", "ṝ" => "r",
+			"ȑ" => "r", "ɾ" => "r", "ᵳ" => "r", "ȓ" => "r", "ṟ" => "r", "ɼ" => "r", "ᵲ" => "r",
+			"ᶉ" => "r", "ɍ" => "r", "ɽ" => "r", "ↄ" => "c", "ꜿ" => "c", "ɘ" => "e", "ɿ" => "r",
+			"ś" => "s", "ṥ" => "s", "š" => "s", "ṧ" => "s", "ş" => "s", "ŝ" => "s", "ș" => "s",
+			"ṡ" => "s", "ṣ" => "s", "ṩ" => "s", "ʂ" => "s", "ᵴ" => "s", "ᶊ" => "s", "ȿ" => "s",
+			"ɡ" => "g", "ᴑ" => "o", "ᴓ" => "o", "ᴝ" => "u", "ť" => "t", "ţ" => "t", "ṱ" => "t",
+			"ț" => "t", "ȶ" => "t", "ẗ" => "t", "ⱦ" => "t", "ṫ" => "t", "ṭ" => "t", "ƭ" => "t",
+			"ṯ" => "t", "ᵵ" => "t", "ƫ" => "t", "ʈ" => "t", "ŧ" => "t", "ᵺ" => "th", "ɐ" => "a",
+			"ᴂ" => "ae", "ǝ" => "e", "ᵷ" => "g", "ɥ" => "h", "ʮ" => "h", "ʯ" => "h", "ᴉ" => "i",
+			"ʞ" => "k", "ꞁ" => "l", "ɯ" => "m", "ɰ" => "m", "ᴔ" => "oe", "ɹ" => "r", "ɻ" => "r",
+			"ɺ" => "r", "ⱹ" => "r", "ʇ" => "t", "ʌ" => "v", "ʍ" => "w", "ʎ" => "y", "ꜩ" => "tz",
+			"ú" => "u", "ŭ" => "u", "ǔ" => "u", "û" => "u", "ṷ" => "u", "ü" => "ue", "ǘ" => "u",
+			"ǚ" => "u", "ǜ" => "u", "ǖ" => "u", "ṳ" => "u", "ụ" => "u", "ű" => "u", "ȕ" => "u",
+			"ù" => "u", "ủ" => "u", "ư" => "u", "ứ" => "u", "ự" => "u", "ừ" => "u", "ử" => "u",
+			"ữ" => "u", "ȗ" => "u", "ū" => "u", "ṻ" => "u", "ų" => "u", "ᶙ" => "u", "ů" => "u",
+			"ũ" => "u", "ṹ" => "u", "ṵ" => "u", "ᵫ" => "ue", "ꝸ" => "um", "ⱴ" => "v", "ꝟ" => "v",
+			"ṿ" => "v", "ʋ" => "v", "ᶌ" => "v", "ⱱ" => "v", "ṽ" => "v", "ꝡ" => "vy", "ẃ" => "w",
+			"ŵ" => "w", "ẅ" => "w", "ẇ" => "w", "ẉ" => "w", "ẁ" => "w", "ⱳ" => "w", "ẘ" => "w",
+			"ẍ" => "x", "ẋ" => "x", "ᶍ" => "x", "ý" => "y", "ŷ" => "y", "ÿ" => "y", "ẏ" => "y",
+			"ỵ" => "y", "ỳ" => "y", "ƴ" => "y", "ỷ" => "y", "ỿ" => "y", "ȳ" => "y", "ẙ" => "y",
+			"ɏ" => "y", "ỹ" => "y", "ź" => "z", "ž" => "z", "ẑ" => "z", "ʑ" => "z", "ⱬ" => "z",
+			"ż" => "z", "ẓ" => "z", "ȥ" => "z", "ẕ" => "z", "ᵶ" => "z", "ᶎ" => "z", "ʐ" => "z",
+			"ƶ" => "z", "ɀ" => "z", "ff" => "ff", "ffi" => "ffi", "ffl" => "ffl", "fi" => "fi",
+			"fl" => "fl", "ij" => "ij", "œ" => "oe", "st" => "st", "ₐ" => "a", "ₑ" => "e", "ᵢ" => "i",
+			"ⱼ" => "j", "ₒ" => "o", "ᵣ" => "r", "ᵤ" => "u", "ᵥ" => "v", "ₓ" => "x"
+		];
+
+		/**
+		 * @see http://en.wikipedia.org/wiki/ISO_843
+		 * @see https://en.wikipedia.org/wiki/Romanization_of_Greek
+		 *
+		 * Resolving diacritic letters can be done using it with option (
+		 * Transliterator::DIACRITICS | Transliterator::GREEK )
+		 *
+		 * @var array
+		 */
+	private static $greekMap = [
+
+			"ου" => "ou",
+			"Ου" => "Ou",
+			"Οι" => "Oi",
+			"οι" => "oi",
+			"Υι" => "Yi",
+			"υι" => "yi",
+			"Ού" => "Oú",
+			"ού" => "oú",
+
+			"Α" => "A",
+			"Ά" => "Á",
+			"Β" => "V",
+			"Γ" => "G",
+			"Δ" => "D",
+			"Ε" => "E",
+			"Έ" => "É",
+			"Ζ" => "Z",
+			"Η" => "Ī",
+			"Ή" => "Ī́",
+			"Θ" => "Th",
+			"Ι" => "I",
+			"Ί" => "Í",
+			"Ϊ" => "Ï",
+			"ΐ" => "ḯ",
+			"Κ" => "K",
+			"Λ" => "L",
+			"Μ" => "M",
+			"Ν" => "N",
+			"Ξ" => "X",
+			"Ο" => "O",
+			"Ό" => "Ó",
+			"Π" => "P",
+			"Ρ" => "R",
+			"Σ" => "S",
+			"ς" => "s",
+			"Τ" => "T",
+			"Υ" => "Y",
+			"Ύ" => "Ý",
+			"Ϋ" => "Ÿ",
+			"ΰ" => "ÿ́",
+			"Φ" => "F",
+			"Χ" => "Ch",
+			"Ψ" => "Ps",
+			"Ω" => "Ō",
+			"Ώ" => "Ṓ",
+			"α" => "a",
+			"ά" => "á",
+			"β" => "v",
+			"γ" => "g",
+			"δ" => "d",
+			"ε" => "e",
+			"έ" => "é",
+			"ζ" => "z",
+			"η" => "ī",
+			"ή" => "ī́",
+			"θ" => "th",
+			"ι" => "i",
+			"ί" => "í",
+			"ϊ" => "ï",
+			"κ" => "k",
+			"λ" => "l",
+			"μ" => "m",
+			"ν" => "n",
+			"ξ" => "x",
+			"ο" => "o",
+			"ό" => "ó",
+			"π" => "p",
+			"ρ" => "r",
+			"σ" => "s",
+			"τ" => "t",
+			"υ" => "y",
+			"ύ" => "ý",
+			"ϋ" => "ÿ",
+			"φ" => "f",
+			"χ" => "ch",
+			"ψ" => "ps",
+			"ω" => "ō",
+			"ώ" => "ṓ",
+		];
+
+	/**
+	 * @var array
+	 */
+	private static $transliterationMap = [];
+
+	/**
+	 * @since 0.1
+	 *
+	 * @param string $string
+	 * @param int $flag
+	 *
+	 * @return string
+	 */
+	public static function transliterate( $string, $flag = self::DIACRITICS ) {
+		$flag = (int)$flag;
+
+		if ( $flag === ( $flag | self::NONE ) ) {
+			return $string;
+		}
+
+		if ( !isset( self::$transliterationMap[$flag] ) ) {
+			$transliterationMap = [];
+
+			if ( $flag === ( $flag | self::GREEK ) ) {
+				$transliterationMap += self::$greekMap;
+			}
+
+			if ( $flag === ( $flag | self::DIACRITICS ) ) {
+				$transliterationMap += self::$diacriticsMap;
+			}
+
+			// Keep it static for the duration of a request to avoid negative
+			// performance impact
+			self::$transliterationMap[$flag]['k'] = array_keys( $transliterationMap );
+			self::$transliterationMap[$flag]['v'] = array_values( $transliterationMap );
+
+			unset( $transliterationMap );
+		}
+
+		return str_replace(
+			self::$transliterationMap[$flag]['k'],
+			self::$transliterationMap[$flag]['v'],
+			$string
+		);
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/tests/bootstrap.php

@@ -0,0 +1,15 @@
+<?php
+
+/**
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+
+error_reporting( E_ALL | E_STRICT );
+date_default_timezone_set( 'UTC' );
+
+if ( PHP_SAPI !== 'cli' ) {
+	die( 'Not an entry point' );
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/tests/phpunit/Fixtures/StopwordAnalyzer/invalid.json

@@ -0,0 +1,5 @@
+{
+	"@source": "Foo",
+	"version": "0.1",
+	"invalid": [],
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/tests/phpunit/Fixtures/StopwordAnalyzer/missingindex.json

@@ -0,0 +1,4 @@
+{
+	"@source": "Foo",
+	"version": "0.1"
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/tests/phpunit/Integration/CombinedNGramTokenizerTest.php

@@ -0,0 +1,107 @@
+<?php
+
+namespace Onoi\Tesa\Tests\Integration;
+
+use Onoi\Tesa\SanitizerFactory;
+use PHPUnit\Framework\TestCase;
+
+/**
+ * @group onoi-tesa
+ *
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class CombinedNGramTokenizerTest extends TestCase {
+
+	/**
+	 * @dataProvider textProvider
+	 */
+	public function testNgramWithBeginEndMarker( $languageCode, $ngramSize, $text, $expected ) {
+		$sanitizerFactory = new SanitizerFactory();
+
+		$tokenizer = $sanitizerFactory->newNGramTokenizer(
+			$sanitizerFactory->newGenericRegExTokenizer()
+		);
+
+		$tokenizer->withMarker( true );
+		$tokenizer->setNgramSize( $ngramSize );
+
+		$tokens = $tokenizer->tokenize( $text );
+
+		$this->assertEquals(
+			$expected,
+			$tokens
+		);
+	}
+
+	public function textProvider() {
+		// https://en.wikipedia.org/wiki/Stop_words
+		$provider[] = [
+			'en',
+			'2',
+			//
+			'In computing, stop words are words which are filtered ...',
+			//
+			[
+				0 => '_i',
+				1 => 'in',
+				2 => 'n_',
+				3 => '_c',
+				4 => 'co',
+				5 => 'om',
+				6 => 'mp',
+				7 => 'pu',
+				8 => 'ut',
+				9 => 'ti',
+				10 => 'in',
+				11 => 'ng',
+				12 => 'g_',
+				13 => '_s',
+				14 => 'st',
+				15 => 'to',
+				16 => 'op',
+				17 => 'p_',
+				18 => '_w',
+				19 => 'wo',
+				20 => 'or',
+				21 => 'rd',
+				22 => 'ds',
+				23 => 's_',
+				24 => '_a',
+				25 => 'ar',
+				26 => 're',
+				27 => 'e_',
+				28 => '_w',
+				29 => 'wo',
+				30 => 'or',
+				31 => 'rd',
+				32 => 'ds',
+				33 => 's_',
+				34 => '_w',
+				35 => 'wh',
+				36 => 'hi',
+				37 => 'ic',
+				38 => 'ch',
+				39 => 'h_',
+				40 => '_a',
+				41 => 'ar',
+				42 => 're',
+				43 => 'e_',
+				44 => '_f',
+				45 => 'fi',
+				46 => 'il',
+				47 => 'lt',
+				48 => 'te',
+				49 => 'er',
+				50 => 're',
+				51 => 'ed',
+				52 => 'd_',
+			]
+		];
+
+		return $provider;
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/tests/phpunit/Integration/CombinedSanitizerTextStopwordTest.php

@@ -0,0 +1,101 @@
+<?php
+
+namespace Onoi\Tesa\Tests\Integration;
+
+use Onoi\Tesa\SanitizerFactory;
+use PHPUnit\Framework\TestCase;
+
+/**
+ * @group onoi-tesa
+ *
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class CombinedSanitizerTextStopwordTest extends TestCase {
+
+	/**
+	 * @dataProvider textByLanguageProvider
+	 */
+	public function testByLanguage( $languageCode, $text, $expected ) {
+		$sanitizerFactory = new SanitizerFactory();
+
+		$sanitizer = $sanitizerFactory->newSanitizer( $text );
+		$sanitizer->toLowercase();
+
+		$text = $sanitizer->sanitizeWith(
+			$sanitizerFactory->newGenericRegExTokenizer(),
+			$sanitizerFactory->newCdbStopwordAnalyzer( $languageCode ),
+			$sanitizerFactory->newNullSynonymizer()
+		);
+
+		$this->assertEquals(
+			$expected,
+			$text
+		);
+	}
+
+	public function textByLanguageProvider() {
+		// https://en.wikipedia.org/wiki/Stop_words
+		$provider[] = [
+			'en',
+			//
+			'In computing, stop words are words which are filtered out before or after processing of ' .
+			'natural language data (text).[1] Though stop words usually refer to the most common words ' .
+			'in a language, there is no single universal list of stop words used by all natural language ' .
+			'processing tools, and indeed not all tools even use such a list. Some tools specifically avoid ' .
+			'removing these stop words to support phrase search.',
+			//
+			'computing stop words filtered processing natural language data text stop words refer common ' .
+			'words language single universal list stop words natural language processing tools list tools ' .
+			'specifically avoid removing stop words support phrase search'
+		];
+
+		// https://es.wikipedia.org/wiki/Palabra_vac%C3%ADa
+		$provider[] = [
+			'es',
+			//
+			'Palabras vacías es el nombre que reciben las palabras sin significado como artículos, pronombres, ' .
+			'preposiciones, etc. que son filtradas antes o después del procesamiento de datos en lenguaje natural ' .
+			'(texto). A Hans Peter Luhn, uno de los pioneros en recuperación de información, se le atribuye la ' .
+			'acuñación de la locución inglesa stop words y el uso del concepto en su diseño. Está controlada por ' .
+			'introducción humana y no automática.',
+			//
+			'palabras vacías nombre que reciben palabras significado artículos pronombres preposiciones etc que ' .
+			'son filtradas después del procesamiento datos lenguaje natural texto hans peter luhn pioneros ' .
+			'recuperación información atribuye acuñación locución inglesa stop words del concepto diseño está ' .
+			'controlada introducción humana automática'
+		];
+
+		// https://de.wikipedia.org/wiki/Stoppwort
+		$provider[] = [
+			'de',
+			//
+			'Stoppwörter nennt man im Information Retrieval Wörter, die bei einer Volltextindexierung nicht beachtet ' .
+			'werden, da sie sehr häufig auftreten und gewöhnlich keine Relevanz für die Erfassung des Dokumentinhalts ' .
+			'besitzen.',
+			//
+			'stoppwörter nennt information retrieval wörter volltextindexierung beachtet häufig auftreten gewöhnlich ' .
+			'relevanz erfassung dokumentinhalts besitzen'
+		];
+
+		// https://en.wikipedia.org/wiki/Query_expansion
+		$provider[] = [
+			'en',
+			//
+			"The goal of query expansion in this regard is by increasing recall, precision can potentially increase " .
+			"(rather than decrease as mathematically equated), by including in the result set pages which are more " .
+			"relevant (of higher quality), or at least equally relevant. Pages which would not be included in the " .
+			"result set, which have the potential to be more relevant to the user's desired query, are included, and " .
+			"without query expansion would not have, regardless of relevance.",
+			//
+			"goal query expansion regard increasing recall precision potentially increase decrease mathematically " .
+			"equated including result set pages relevant quality equally relevant pages included result set potential " .
+			"relevant user desired query included query expansion relevance"
+		];
+
+		return $provider;
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/tests/phpunit/Integration/JaTokenizerTest.php

@@ -0,0 +1,115 @@
+<?php
+
+namespace Onoi\Tesa\Tests\Integration;
+
+use Onoi\Tesa\SanitizerFactory;
+use PHPUnit\Framework\TestCase;
+
+/**
+ * @group onoi-tesa
+ *
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class JaTokenizerTest extends TestCase {
+
+	/**
+	 * @dataProvider icuTextProvider
+	 */
+	public function testIcuWordBoundaryTokenizer( $text, $expected ) {
+		$sanitizerFactory = new SanitizerFactory();
+
+		$tokenier = $sanitizerFactory->newIcuWordBoundaryTokenizer(
+			$sanitizerFactory->newGenericRegExTokenizer()
+		);
+
+		if ( !$tokenier->isAvailable() ) {
+			$this->markTestSkipped( 'ICU extension is not available.' );
+		}
+
+		$this->assertEquals(
+			$expected,
+			$tokenier->tokenize( $text )
+		);
+	}
+
+	/**
+	 * @dataProvider tinyTextProvider
+	 */
+	public function testJaTinySegmenterTokenizer( $text, $expected ) {
+		$sanitizerFactory = new SanitizerFactory();
+
+		$tokenier = $sanitizerFactory->newJaTinySegmenterTokenizer(
+			 $sanitizerFactory->newPunctuationRegExTokenizer()
+		);
+
+		$this->assertEquals(
+			$expected,
+			$tokenier->tokenize( $text )
+		);
+	}
+
+	public function icuTextProvider() {
+		// https://github.com/NaturalNode/natural/blob/master/spec/tokenizer_ja_spec.js
+
+		$provider[] = [
+			"計算機科学における字句解析 (じくかいせき、英: Lexical Analysis) とは、ソースコードを構成する文字の並びを、トークン (token) の並びに変換することをいう。\n" .
+			"ここでいう「トークン」とは、意味を持つコードの最小単位のこと。字句解析を行うプログラムは、字句解析器 (lexical analyzer, 略称：lexer) と呼ばれる。\n" .
+			"字句解析器はスキャナ (scanner) とトークナイザ (tokenizer) から構成される。\n",
+			//
+			[
+				'計算機', '科学', 'における', '字句', '解析', 'じ', 'く', 'かい',
+				'せき', '、', '英', 'Lexical', 'Analysis', 'と', 'は', '、', 'ソース', 'コード', 'を',
+				'構成', 'する', '文字', 'の', '並び', 'を', '、', 'トーク',
+				'ン', 'token', 'の', '並びに', '変換', 'する', 'こと', 'を',
+				'いう', '。', 'ここ', 'で', 'いう', '「', 'トーク', 'ン',
+				'」', 'と', 'は', '、', '意味', 'を', '持つ', 'コード',
+				'の', '最小', '単位', 'の', 'こと', '。', '字句', '解析',
+				'を', '行う', 'プログラム', 'は', '、', '字句', '解析', '器',
+				'lexical', 'analyzer', '略称', '：',
+				'lexer', 'と', '呼ばれる', '。', '字句', '解析', '器', 'は',
+				'スキャナ', 'scanner', 'と', 'トーク', 'ナ', 'イザ', 'tokenizer', 'から',
+				'構成', 'さ', 'れる', '。'
+			]
+		];
+
+		return $provider;
+	}
+
+	public function tinyTextProvider() {
+		// https://github.com/NaturalNode/natural/blob/master/spec/tokenizer_ja_spec.js
+		/*
+			['計算', '機科', '学', 'に', 'おける', '字句', '解析',
+			'じくかい', 'せき', '英', 'Lexical', 'Analysis', 'と', 'は', 'ソースコード',
+			'を', '構成', 'する', '文字', 'の', '並び', 'を', 'トークン', 'token', 'の',
+			'並び', 'に', '変換', 'する', 'こと', 'を', 'いう', 'ここ', 'でいう', 'トークン',
+			'と', 'は', '意味', 'を', '持つ', 'コード', 'の', '最小', '単位', 'の', 'こと',
+			'字句', '解析', 'を', '行う', 'プログラム', 'は', '字句', '解析', '器', 'lexical',
+			'analyzer', '略称', 'lexer', 'と', '呼ば', 'れる', '字句', '解析', '器', 'は',
+			'スキャナ', 'scanner', 'と', 'トークナイザ', 'tokenizer', 'から', '構成', 'さ',
+			'れる']
+		 */
+
+		$provider[] = [
+			"計算機科学における字句解析 (じくかいせき、英: Lexical Analysis) とは、ソースコードを構成する文字の並びを、トークン (token) の並びに変換することをいう。\n" .
+			"ここでいう「トークン」とは、意味を持つコードの最小単位のこと。字句解析を行うプログラムは、字句解析器 (lexical analyzer, 略称：lexer) と呼ばれる。\n" .
+			"字句解析器はスキャナ (scanner) とトークナイザ (tokenizer) から構成される。\n",
+			//
+			[
+				'計算', '機科', '学', 'に', 'おける', '字句', '解析', 'じくかい', 'せき', '英',
+				'Lexical', 'Analysis', 'と', 'は', 'ソースコード', 'を', '構成', 'する',
+				'文字', 'の', '並び', 'を', 'トークン', 'token', 'の', '並び', 'に', '変換',
+				'する', 'こと', 'をいう', 'ここ', 'でいう', 'トークン', 'と', 'は', '意味', 'を',
+				'持つ', 'コード', 'の', '最小', '単位', 'の', 'こと', '字句', '解析', 'を',
+				'行う', 'プログラム', 'は', '字句', '解析', '器', 'lexical', 'analyzer',
+				'略称', 'lexer', 'と', '呼ば', 'れる', '字句', '解析', '器', 'は', 'スキャナ', 'scanner',
+				'と', 'トークナイザ', 'tokenizer', 'から', '構成', 'さ', 'れる',
+			]
+		];
+
+		return $provider;
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/tests/phpunit/Unit/CharacterExaminerTest.php

@@ -0,0 +1,73 @@
+<?php
+
+namespace Onoi\Tesa\Tests;
+
+use Onoi\Tesa\CharacterExaminer;
+use PHPUnit\Framework\TestCase;
+
+/**
+ * @covers \Onoi\Tesa\CharacterExaminer
+ * @group onoi-tesa
+ *
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class CharacterExaminerTest extends TestCase {
+
+	public function testToContainKoreanCharacters() {
+		$this->assertTrue(
+			CharacterExaminer::contains( CharacterExaminer::HANGUL, '한국어 텍스트의 예' )
+		);
+
+		$this->assertFalse(
+			CharacterExaminer::contains( CharacterExaminer::HAN, '한국어 텍스트의 예' )
+		);
+	}
+
+	public function testToContainJapaneseCharacters() {
+		$this->assertTrue(
+			CharacterExaminer::contains( CharacterExaminer::LATIN, '脳のIQテスト' )
+		);
+
+		$this->assertTrue(
+			CharacterExaminer::contains( CharacterExaminer::HIRAGANA_KATAKANA, '脳のIQテスト' )
+		);
+
+		$this->assertTrue(
+			CharacterExaminer::contains( CharacterExaminer::HAN, '脳のIQテスト' )
+		);
+	}
+
+	public function testToContainChineseCharacters() {
+		$this->assertFalse(
+			CharacterExaminer::contains( CharacterExaminer::LATIN, '才可以过关' )
+		);
+
+		$this->assertTrue(
+			CharacterExaminer::contains( CharacterExaminer::CJK_UNIFIED, '才可以过关' )
+		);
+
+		$this->assertTrue(
+			CharacterExaminer::contains( CharacterExaminer::HAN, '才可以过关' )
+		);
+	}
+
+	public function testToContainCyrillic() {
+		$this->assertFalse(
+			CharacterExaminer::contains( CharacterExaminer::LATIN, 'Привет' )
+		);
+
+		$this->assertTrue(
+			CharacterExaminer::contains( CharacterExaminer::CYRILLIC, 'Привет' )
+		);
+	}
+
+	public function testToContainUnknownCharacters() {
+		$this->assertFalse(
+			CharacterExaminer::contains( 'Foo', '鿩' )
+		);
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/tests/phpunit/Unit/LanguageDetector/NullLanguageDetectorTest.php

@@ -0,0 +1,34 @@
+<?php
+
+namespace Onoi\Tesa\Tests\LanguageDetector;
+
+use Onoi\Tesa\LanguageDetector\NullLanguageDetector;
+use PHPUnit\Framework\TestCase;
+
+/**
+ * @covers \Onoi\Tesa\LanguageDetector\NullLanguageDetector
+ * @group onoi-tesa
+ *
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class NullLanguageDetectorTest extends TestCase {
+
+	public function testCanConstruct() {
+		$this->assertInstanceOf(
+			'\Onoi\Tesa\LanguageDetector\NullLanguageDetector',
+			new NullLanguageDetector()
+		);
+	}
+
+	public function testIsStopWord() {
+		$instance = new NullLanguageDetector();
+
+		$this->assertNull(
+			$instance->detect( 'Foo' )
+		);
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/tests/phpunit/Unit/LanguageDetector/TextCatLanguageDetectorTest.php

@@ -0,0 +1,48 @@
+<?php
+
+namespace Onoi\Tesa\Tests\LanguageDetector;
+
+use Onoi\Tesa\LanguageDetector\TextCatLanguageDetector;
+use PHPUnit\Framework\TestCase;
+
+/**
+ * @covers \Onoi\Tesa\LanguageDetector\TextCatLanguageDetector
+ * @group onoi-tesa
+ *
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class TextCatLanguageDetectorTest extends TestCase {
+
+	public function testCanConstruct() {
+		$this->assertInstanceOf(
+			'\Onoi\Tesa\LanguageDetector\TextCatLanguageDetector',
+			new TextCatLanguageDetector()
+		);
+	}
+
+	public function testDetectOnMock() {
+		$languageCandidates = [ 'en', 'de', 'fr', 'es', 'ja', 'zh' ];
+
+		$textCat = $this->getMockBuilder( '\TextCat' )
+			->disableOriginalConstructor()
+			->getMock();
+
+		$textCat->expects( $this->once() )
+			->method( 'classify' )
+			->with(
+				'Foo',
+				$languageCandidates )
+			->willReturn( [] );
+
+		$instance = new TextCatLanguageDetector( $textCat );
+		$instance->setLanguageCandidates(
+			$languageCandidates
+		);
+
+		$instance->detect( 'Foo' );
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/tests/phpunit/Unit/NormalizerTest.php

@@ -0,0 +1,72 @@
+<?php
+
+namespace Onoi\Tesa\Tests;
+
+use Onoi\Tesa\Normalizer;
+use PHPUnit\Framework\TestCase;
+
+/**
+ * @covers \Onoi\Tesa\Normalizer
+ * @group onoi-tesa
+ *
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class NormalizerTest extends TestCase {
+
+	public function testTransliteration() {
+		$this->assertEquals(
+			'AAAAAEAaaaaaeaOOOOOOEOoooooeoEEEEeeeeðCcÐIIIIiiiiUUUUEuuuueNnSsYyyZz',
+			Normalizer::applyTransliteration( 'ÀÁÂÃÄÅàáâãäåÒÓÔÕÕÖØòóôõöøÈÉÊËèéêëðÇçÐÌÍÎÏìíîïÙÚÛÜùúûüÑñŠšŸÿýŽž' )
+		);
+	}
+
+	public function testConvertDoubleWidth() {
+		$this->assertSame(
+			'0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz',
+			Normalizer::convertDoubleWidth( '０１２３４５６７８９ＡＢＣＤＥＦＧＨＩＪＫＬＭＮＯＰＱＲＳＴＵＶＷＸＹＺａｂｃｄｅｆｇｈｉｊｋｌｍｎｏｐｑｒｓｔｕｖｗｘｙｚ' )
+		);
+	}
+
+	public function testReduceLengthTo() {
+		$this->assertEquals(
+			'ABC',
+			Normalizer::reduceLengthTo( 'ABCDEF', 3 )
+		);
+
+		$this->assertEquals(
+			'ABCDEF',
+			Normalizer::reduceLengthTo( 'ABCDEF' )
+		);
+
+		$this->assertEquals(
+			'ABCD',
+			Normalizer::reduceLengthTo( 'ABCD EF', 4 )
+		);
+
+		$this->assertEquals(
+			'ABC',
+			Normalizer::reduceLengthTo( 'ABC D EF', 4 )
+		);
+
+		$this->assertEquals(
+			'ABCD',
+			Normalizer::reduceLengthTo( 'ABCD EF', 5 )
+		);
+
+		$this->assertEquals(
+			'abc def gh',
+			Normalizer::reduceLengthTo( 'abc def gh in 123', 12 )
+		);
+	}
+
+	public function testToLowercase() {
+		$this->assertEquals(
+			'abcdef',
+			Normalizer::toLowercase( 'ABCDEF' )
+		);
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/tests/phpunit/Unit/SanitizerFactoryTest.php

@@ -0,0 +1,176 @@
+<?php
+
+namespace Onoi\Tesa\Tests;
+
+use Onoi\Tesa\SanitizerFactory;
+use PHPUnit\Framework\TestCase;
+
+/**
+ * @covers \Onoi\Tesa\SanitizerFactory
+ * @group onoi-tesa
+ *
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class SanitizerFactoryTest extends TestCase {
+
+	private $sanitizerFactory;
+
+	protected function setUp(): void {
+		$this->sanitizerFactory = new SanitizerFactory();
+	}
+
+	public function testCanConstructSanitizer() {
+		$this->assertInstanceOf(
+			'\Onoi\Tesa\Sanitizer',
+			$this->sanitizerFactory->newSanitizer()
+		);
+	}
+
+	/* StopwordAnalyzer */
+
+	public function testCanConstructStopwordAnalyzerByNullLanguage() {
+		$this->assertInstanceOf(
+			'\Onoi\Tesa\StopwordAnalyzer\StopwordAnalyzer',
+			$this->sanitizerFactory->newStopwordAnalyzerByLanguage( null )
+		);
+	}
+
+	public function testCanConstructStopwordAnalyzerByEnLanguage() {
+		$this->assertInstanceOf(
+			'\Onoi\Tesa\StopwordAnalyzer\CdbStopwordAnalyzer',
+			$this->sanitizerFactory->newStopwordAnalyzerByLanguage( 'EN' )
+		);
+	}
+
+	public function testCanConstructCdbStopwordAnalyzer() {
+		$this->assertInstanceOf(
+			'\Onoi\Tesa\StopwordAnalyzer\CdbStopwordAnalyzer',
+			$this->sanitizerFactory->newCdbStopwordAnalyzer()
+		);
+	}
+
+	public function testCanConstructArrayStopwordAnalyzer() {
+		$this->assertInstanceOf(
+			'\Onoi\Tesa\StopwordAnalyzer\ArrayStopwordAnalyzer',
+			$this->sanitizerFactory->newArrayStopwordAnalyzer()
+		);
+	}
+
+	public function testCanConstructNullStopwordAnalyzer() {
+		$this->assertInstanceOf(
+			'\Onoi\Tesa\StopwordAnalyzer\NullStopwordAnalyzer',
+			$this->sanitizerFactory->newNullStopwordAnalyzer()
+		);
+	}
+
+	/* Synonymizer */
+
+	public function testCanConstructSynonymizerByLanguage() {
+		$this->assertInstanceOf(
+			'\Onoi\Tesa\Synonymizer\Synonymizer',
+			$this->sanitizerFactory->newSynonymizerByLanguage()
+		);
+	}
+
+	public function testCanConstructNullSynonymizer() {
+		$this->assertInstanceOf(
+			'\Onoi\Tesa\Synonymizer\NullSynonymizer',
+			$this->sanitizerFactory->newNullSynonymizer()
+		);
+	}
+
+	/* LanguageDetector */
+
+	public function testCanConstructNullLanguageDetector() {
+		$this->assertInstanceOf(
+			'\Onoi\Tesa\LanguageDetector\NullLanguageDetector',
+			$this->sanitizerFactory->newNullLanguageDetector()
+		);
+	}
+
+	public function testCanConstructTextCatLanguageDetector() {
+		$this->assertInstanceOf(
+			'\Onoi\Tesa\LanguageDetector\TextCatLanguageDetector',
+			$this->sanitizerFactory->newTextCatLanguageDetector()
+		);
+	}
+
+	/* Tokenizer */
+
+	public function testCanConstructPreferredTokenizerByLanguage() {
+		$this->assertInstanceOf(
+			'\Onoi\Tesa\Tokenizer\Tokenizer',
+			$this->sanitizerFactory->newPreferredTokenizerByLanguage( 'テスト' )
+		);
+
+		$this->assertInstanceOf(
+			'\Onoi\Tesa\Tokenizer\Tokenizer',
+			$this->sanitizerFactory->newPreferredTokenizerByLanguage( '在延安更名为新' )
+		);
+	}
+
+	public function testCanConstructnewCJKMatchableTokenizer() {
+		$this->assertInstanceOf(
+			'\Onoi\Tesa\Tokenizer\Tokenizer',
+			$this->sanitizerFactory->newCJKMatchableTokenizer( 'テスト' )
+		);
+
+		$this->assertInstanceOf(
+			'\Onoi\Tesa\Tokenizer\Tokenizer',
+			$this->sanitizerFactory->newCJKMatchableTokenizer( '在延安更名为新' )
+		);
+	}
+
+	public function testCanConstructIcuWordBoundaryTokenizer() {
+		$this->assertInstanceOf(
+			'\Onoi\Tesa\Tokenizer\IcuWordBoundaryTokenizer',
+			$this->sanitizerFactory->newIcuWordBoundaryTokenizer()
+		);
+	}
+
+	public function testCanConstructGenericRegExTokenizer() {
+		$this->assertInstanceOf(
+			'\Onoi\Tesa\Tokenizer\GenericRegExTokenizer',
+			$this->sanitizerFactory->newGenericRegExTokenizer()
+		);
+	}
+
+	public function testCanConstructPunctuationRegExTokenizer() {
+		$this->assertInstanceOf(
+			'\Onoi\Tesa\Tokenizer\PunctuationRegExTokenizer',
+			$this->sanitizerFactory->newPunctuationRegExTokenizer()
+		);
+	}
+
+	public function testCanConstructJaCompoundGroupTokenizer() {
+		$this->assertInstanceOf(
+			'\Onoi\Tesa\Tokenizer\JaCompoundGroupTokenizer',
+			$this->sanitizerFactory->newJaCompoundGroupTokenizer()
+		);
+	}
+
+	public function testCanConstructJaTinySegmenterTokenizer() {
+		$this->assertInstanceOf(
+			'\Onoi\Tesa\Tokenizer\JaTinySegmenterTokenizer',
+			$this->sanitizerFactory->newJaTinySegmenterTokenizer()
+		);
+	}
+
+	public function testCanConstructCJKSimpleCharacterRegExTokenizer() {
+		$this->assertInstanceOf(
+			'\Onoi\Tesa\Tokenizer\CJKSimpleCharacterRegExTokenizer',
+			$this->sanitizerFactory->newCJKSimpleCharacterRegExTokenizer()
+		);
+	}
+
+	public function testCanConstructNGramTokenizer() {
+		$this->assertInstanceOf(
+			'\Onoi\Tesa\Tokenizer\NGramTokenizer',
+			$this->sanitizerFactory->newNGramTokenizer()
+		);
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/tests/phpunit/Unit/SanitizerTest.php

@@ -0,0 +1,208 @@
+<?php
+
+namespace Onoi\Tesa\Tests;
+
+use Onoi\Tesa\Sanitizer;
+use Onoi\Tesa\SanitizerFactory;
+use PHPUnit\Framework\TestCase;
+
+/**
+ * @covers \Onoi\Tesa\Sanitizer
+ * @group onoi-tesa
+ *
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class SanitizerTest extends TestCase {
+
+	private $sanitizerFactory;
+
+	protected function setUp(): void {
+		$this->sanitizerFactory = new SanitizerFactory();
+	}
+
+	public function testTransliteration() {
+		$instance = new Sanitizer( 'ÀÁÂÃÄÅàáâãäåÒÓÔÕÕÖØòóôõöøÈÉÊËèéêëðÇçÐÌÍÎÏìíîïÙÚÛÜùúûüÑñŠšŸÿýŽž' );
+		$instance->applyTransliteration();
+
+		$this->assertEquals(
+			'AAAAAEAaaaaaeaOOOOOOEOoooooeoEEEEeeeeðCcÐIIIIiiiiUUUUEuuuueNnSsYyyZz',
+			$instance
+		);
+	}
+
+	public function testToLowercase() {
+		$instance = new Sanitizer( 'ÀÁÂÃÄÅ ABC 텍스트의 テスト часто הוא פשוט' );
+		$instance->toLowercase();
+
+		$this->assertEquals(
+			'àáâãäå abc 텍스트의 テスト часто הוא פשוט',
+			$instance
+		);
+	}
+
+	public function testReduceLengthTo() {
+		$instance = new Sanitizer( 'ABCDEF' );
+		$instance->reduceLengthTo( 3 );
+
+		$this->assertEquals(
+			3,
+			mb_strlen( $instance )
+		);
+
+		$instance->reduceLengthTo( 10 );
+
+		$this->assertEquals(
+			3,
+			mb_strlen( $instance )
+		);
+	}
+
+	public function testReduceLengthToNearestWholeWordForLatinString() {
+		$instance = new Sanitizer( 'abc def gh in 123' );
+		$instance->reduceLengthTo( 12 );
+
+		$this->assertEquals(
+			10,
+			mb_strlen( $instance )
+		);
+
+		$this->assertEquals(
+			'abc def gh',
+			$instance
+		);
+	}
+
+	public function testReduceLengthToNearestWholeWordForNonLatinString() {
+		$instance = new Sanitizer( '一　二　三' );
+		$instance->reduceLengthTo( 3 );
+
+		$this->assertEquals(
+			3,
+			mb_strlen( $instance )
+		);
+
+		$this->assertEquals(
+			'一　二',
+			$instance
+		);
+	}
+
+	public function testReplace() {
+		$instance = new Sanitizer( 'テスト' );
+		$instance->replace( [ 'テスト' ], [ 'Test' ] );
+
+		$this->assertEquals(
+			'Test',
+			$instance
+		);
+	}
+
+	public function testSanitizeWithSimpleStopwordList() {
+		$text = 'Foo bar foobar';
+
+		$tokenizer = $this->getMockBuilder( '\Onoi\Tesa\Tokenizer\Tokenizer' )
+			->disableOriginalConstructor()
+			->getMockForAbstractClass();
+
+		$tokenizer->expects( $this->once() )
+			->method( 'tokenize' )
+			->with( $text )
+			->willReturn( [ 'Foo', 'bar', 'foobar' ] );
+
+		$synonymizer = $this->getMockBuilder( '\Onoi\Tesa\Synonymizer\Synonymizer' )
+			->disableOriginalConstructor()
+			->getMockForAbstractClass();
+
+		$synonymizer->expects( $this->any() )
+			->method( 'synonymize' )
+			->willReturnArgument( 0 );
+
+		$instance = new Sanitizer( $text );
+
+		$stopwordAnalyzer = $this->sanitizerFactory->newArrayStopwordAnalyzer(
+			[ 'bar' ]
+		);
+
+		$this->assertEquals(
+			'Foo foobar',
+			$instance->sanitizeWith( $tokenizer, $stopwordAnalyzer, $synonymizer )
+		);
+	}
+
+	public function testSanitizeByStopwordsToIncludeExemptionWithMinLengthRestriction() {
+		$text = 'Foo bar foobar';
+
+		$tokenizer = $this->getMockBuilder( '\Onoi\Tesa\Tokenizer\Tokenizer' )
+			->disableOriginalConstructor()
+			->getMockForAbstractClass();
+
+		$tokenizer->expects( $this->once() )
+			->method( 'isWordTokenizer' )
+			->willReturn( true );
+
+		$tokenizer->expects( $this->once() )
+			->method( 'tokenize' )
+			->with( $text )
+			->willReturn( [ 'Foo', 'bar', 'foobar' ] );
+
+		$synonymizer = $this->getMockBuilder( '\Onoi\Tesa\Synonymizer\Synonymizer' )
+			->disableOriginalConstructor()
+			->getMockForAbstractClass();
+
+		$synonymizer->expects( $this->any() )
+			->method( 'synonymize' )
+			->willReturnArgument( 0 );
+
+		$instance = new Sanitizer( $text );
+
+		$stopwordAnalyzer = $this->sanitizerFactory->newArrayStopwordAnalyzer(
+			[ 'bar' ]
+		);
+
+		$instance->setOption( Sanitizer::MIN_LENGTH, 4 );
+		$instance->setOption( Sanitizer::WHITELIST, [ 'bar' ] );
+
+		$this->assertEquals(
+			'bar foobar',
+			$instance->sanitizeWith( $tokenizer, $stopwordAnalyzer, $synonymizer )
+		);
+	}
+
+	public function testTrySanitizeByStopwordsWithProximityCheck() {
+		$text = 'foo foo テスト テスト';
+
+		$tokenizer = $this->getMockBuilder( '\Onoi\Tesa\Tokenizer\Tokenizer' )
+			->disableOriginalConstructor()
+			->getMockForAbstractClass();
+
+		$tokenizer->expects( $this->once() )
+			->method( 'isWordTokenizer' )
+			->willReturn( true );
+
+		$tokenizer->expects( $this->once() )
+			->method( 'tokenize' )
+			->with( $text )
+			->willReturn( [ 'foo', 'foo', 'テスト', 'テスト' ] );
+
+		$synonymizer = $this->getMockBuilder( '\Onoi\Tesa\Synonymizer\Synonymizer' )
+			->disableOriginalConstructor()
+			->getMockForAbstractClass();
+
+		$synonymizer->expects( $this->any() )
+			->method( 'synonymize' )
+			->willReturnArgument( 0 );
+
+		$instance = new Sanitizer( $text );
+
+		$stopwordAnalyzer = $this->sanitizerFactory->newArrayStopwordAnalyzer();
+
+		$this->assertEquals(
+			'foo テスト',
+			$instance->sanitizeWith( $tokenizer, $stopwordAnalyzer, $synonymizer )
+		);
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/tests/phpunit/Unit/StopwordAnalyzer/ArrayStopwordAnalyzerTest.php

@@ -0,0 +1,62 @@
+<?php
+
+namespace Onoi\Tesa\Tests\StopwordAnalyzer;
+
+use Onoi\Tesa\StopwordAnalyzer\ArrayStopwordAnalyzer;
+use PHPUnit\Framework\TestCase;
+
+/**
+ * @covers \Onoi\Tesa\StopwordAnalyzer\ArrayStopwordAnalyzer
+ * @group onoi-tesa
+ *
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class ArrayStopwordAnalyzerTest extends TestCase {
+
+	public function testCanConstruct() {
+		$this->assertInstanceOf(
+			'\Onoi\Tesa\StopwordAnalyzer\ArrayStopwordAnalyzer',
+			new ArrayStopwordAnalyzer()
+		);
+	}
+
+	/**
+	 * @dataProvider stopWordsProvider
+	 */
+	public function testIsStopWord( $defaultList, $word, $expected ) {
+		$instance = new ArrayStopwordAnalyzer( $defaultList );
+
+		$this->assertEquals(
+			$expected,
+			$instance->isStopWord( $word )
+		);
+	}
+
+	public function stopWordsProvider() {
+		$defaultList = [ 'Foo', 'かつて', 'bAR' ];
+
+		$provider[] = [
+			$defaultList,
+			'Foo',
+			true
+		];
+
+		$provider[] = [
+			$defaultList,
+			'かつて',
+			true
+		];
+
+		$provider[] = [
+			$defaultList,
+			'bar',
+			false
+		];
+
+		return $provider;
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/tests/phpunit/Unit/StopwordAnalyzer/CdbStopwordAnalyzerTest.php

@@ -0,0 +1,162 @@
+<?php
+
+namespace Onoi\Tesa\Tests\StopwordAnalyzer;
+
+use Onoi\Tesa\StopwordAnalyzer\CdbStopwordAnalyzer;
+use PHPUnit\Framework\TestCase;
+
+/**
+ * @covers \Onoi\Tesa\StopwordAnalyzer\CdbStopwordAnalyzer
+ * @group onoi-tesa
+ *
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class CdbStopwordAnalyzerTest extends TestCase {
+
+	public function testTryToCreateCdbByLanguageOnInvalidLanguageThrowsException() {
+		$this->expectException( 'RuntimeException' );
+
+		CdbStopwordAnalyzer::createCdbByLanguage(
+			CdbStopwordAnalyzer::getLocation(),
+			'foo'
+		);
+	}
+
+	public function testTryToCreateCdbByLanguageOnInvalidJsonIndexThrowsException() {
+		$this->expectException( 'RuntimeException' );
+
+		CdbStopwordAnalyzer::createCdbByLanguage(
+			__DIR__ . '/../../Fixtures/StopwordAnalyzer/',
+			'missingindex'
+		);
+	}
+
+	public function testTryToCreateCdbByLanguageOnInvalidJsonThrowsException() {
+		$this->expectException( 'RuntimeException' );
+
+		CdbStopwordAnalyzer::createCdbByLanguage(
+			__DIR__ . '/../../Fixtures/StopwordAnalyzer/',
+			'invalid'
+		);
+	}
+
+	/**
+	 * @dataProvider languageProvider
+	 */
+	public function testCreateCdbByLanguage( $languageCode ) {
+		$res = CdbStopwordAnalyzer::createCdbByLanguage(
+			CdbStopwordAnalyzer::getLocation(),
+			$languageCode
+		);
+
+		$this->assertTrue(
+			$res
+		);
+	}
+
+	/**
+	 * @dataProvider stopWordProvider
+	 */
+	public function testIsStopWord( $languageCode, $word, $expected ) {
+		$instane = new CdbStopwordAnalyzer(
+			CdbStopwordAnalyzer::getTargetByLanguage( $languageCode )
+		);
+
+		$this->assertEquals(
+			$expected,
+			$instane->isStopWord( $word )
+		);
+	}
+
+	public function languageProvider() {
+		$provider[] = [
+			'en',
+		];
+
+		$provider[] = [
+			'de'
+		];
+
+		$provider[] = [
+			'ja'
+		];
+
+		$provider[] = [
+			'zh'
+		];
+
+		$provider[] = [
+			'es'
+		];
+
+		$provider[] = [
+			'fr'
+		];
+
+		$provider[] = [
+			'pt'
+		];
+
+		$provider[] = [
+			'pt-br'
+		];
+
+		return $provider;
+	}
+
+	public function stopWordProvider() {
+		$provider[] = [
+			'en',
+			'Foo',
+			false
+		];
+
+		$provider[] = [
+			'en',
+			'the',
+			true
+		];
+
+		$provider[] = [
+			'ja',
+			'それぞれ',
+			true
+		];
+
+		$provider[] = [
+			'zh',
+			'不单',
+			true
+		];
+
+		$provider[] = [
+			'es',
+			'arriba',
+			true
+		];
+
+		$provider[] = [
+			'fr',
+			'devrait',
+			true
+		];
+
+		$provider[] = [
+			'pt',
+			'conhecido',
+			true
+		];
+
+		$provider[] = [
+			'pt-br',
+			'mediante',
+			true
+		];
+
+		return $provider;
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/tests/phpunit/Unit/StopwordAnalyzer/NullStopwordAnalyzerTest.php

@@ -0,0 +1,34 @@
+<?php
+
+namespace Onoi\Tesa\Tests\StopwordAnalyzer;
+
+use Onoi\Tesa\StopwordAnalyzer\NullStopwordAnalyzer;
+use PHPUnit\Framework\TestCase;
+
+/**
+ * @covers \Onoi\Tesa\StopwordAnalyzer\NullStopwordAnalyzer
+ * @group onoi-tesa
+ *
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class NullStopwordAnalyzerTest extends TestCase {
+
+	public function testCanConstruct() {
+		$this->assertInstanceOf(
+			'\Onoi\Tesa\StopwordAnalyzer\NullStopwordAnalyzer',
+			new NullStopwordAnalyzer()
+		);
+	}
+
+	public function testIsStopWord() {
+		$instance = new NullStopwordAnalyzer();
+
+		$this->assertFalse(
+			$instance->isStopWord( 'Foo' )
+		);
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/tests/phpunit/Unit/Synonymizer/NullSynonymizerTest.php

@@ -0,0 +1,35 @@
+<?php
+
+namespace Onoi\Tesa\Tests\Synonymizer;
+
+use Onoi\Tesa\Synonymizer\NullSynonymizer;
+use PHPUnit\Framework\TestCase;
+
+/**
+ * @covers \Onoi\Tesa\Synonymizer\NullSynonymizer
+ * @group onoi-tesa
+ *
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class NullSynonymizerTest extends TestCase {
+
+	public function testCanConstruct() {
+		$this->assertInstanceOf(
+			'\Onoi\Tesa\Synonymizer\NullSynonymizer',
+			new NullSynonymizer()
+		);
+	}
+
+	public function testSynonymize() {
+		$instance = new NullSynonymizer();
+
+		$this->assertEquals(
+			'Foo',
+			$instance->synonymize( 'Foo' )
+		);
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/tests/phpunit/Unit/Tokenizer/CJKSimpleCharacterRegExTokenizerTest.php

@@ -0,0 +1,84 @@
+<?php
+
+namespace Onoi\Tesa\Tests;
+
+use Onoi\Tesa\Tokenizer\CJKSimpleCharacterRegExTokenizer;
+use PHPUnit\Framework\TestCase;
+
+/**
+ * @covers \Onoi\Tesa\Tokenizer\CJKSimpleCharacterRegExTokenizer
+ * @group onoi-tesa
+ *
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class CJKSimpleCharacterRegExTokenizerTest extends TestCase {
+
+	public function testUnknownOption() {
+		$this->assertInstanceOf(
+			'\Onoi\Tesa\Tokenizer\CJKSimpleCharacterRegExTokenizer',
+			new CJKSimpleCharacterRegExTokenizer()
+		);
+	}
+
+	/**
+	 * @dataProvider stringProvider
+	 */
+	public function testTokenize( $string, $expected ) {
+		$instance = new CJKSimpleCharacterRegExTokenizer();
+
+		$this->assertEquals(
+			$expected,
+			$instance->tokenize( $string )
+		);
+
+		$this->assertFalse(
+			$instance->isWordTokenizer()
+		);
+	}
+
+	public function testTokenizeWithEnabledExemptionList() {
+		$string = '《红色中华》报改名为《新中华报》的同时，在';
+
+		$tokenizer = $this->getMockBuilder( '\Onoi\Tesa\Tokenizer\Tokenizer' )
+			->disableOriginalConstructor()
+			->getMockForAbstractClass();
+
+		$tokenizer->expects( $this->once() )
+			->method( 'setOption' );
+
+		$tokenizer->expects( $this->once() )
+			->method( 'tokenize' )
+			->with( $string )
+			->willReturn( [ $string ] );
+
+		$instance = new CJKSimpleCharacterRegExTokenizer( $tokenizer );
+
+		$instance->setOption(
+			CJKSimpleCharacterRegExTokenizer::REGEX_EXEMPTION,
+			[ '《', '》', '，' ]
+		);
+
+		$this->assertEquals(
+			[ '《红色中华》报改名', '《新中华报》', '同', '，' ],
+			$instance->tokenize( $string )
+		);
+	}
+
+	public function stringProvider() {
+		$provider[] = [
+			'《红色中华》报改名为《新中华报》的同时，在延安更名为新华通讯社。但是当时，新华社和《新中华报》还是同一个机构。',
+			[ '红色中华', '报改名', '新中华报', '同', '延安更名', '新华通讯社', '新华社', '新中华报', '同一', '机构' ]
+		];
+
+		$provider[] = [
+			'江泽民在北京人民大会堂会见参加全国法院工作会议和全国法院系统打击经济犯罪先进集体表彰大会代表时要求大家要充分认识打击经济犯罪的艰巨性和长期性',
+			[ '江泽民', '北京人民大会堂会见参加全国法院工作会议', '全国法院系统打击经济犯罪先进集体表彰大会代表', '求大家', '充分认识打击经济犯罪', '艰巨性', '长期性' ]
+		];
+
+		return $provider;
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/tests/phpunit/Unit/Tokenizer/GenericRegExTokenizerTest.php

@@ -0,0 +1,91 @@
+<?php
+
+namespace Onoi\Tesa\Tests;
+
+use Onoi\Tesa\Tokenizer\GenericRegExTokenizer;
+use PHPUnit\Framework\TestCase;
+
+/**
+ * @covers \Onoi\Tesa\Tokenizer\GenericRegExTokenizer
+ * @group onoi-tesa
+ *
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class GenericRegExTokenizerTest extends TestCase {
+
+	public function testUnknownOption() {
+		$this->assertInstanceOf(
+			'\Onoi\Tesa\Tokenizer\GenericRegExTokenizer',
+			new GenericRegExTokenizer()
+		);
+	}
+
+	/**
+	 * @dataProvider stringProvider
+	 */
+	public function testTokenize( $string, $expected ) {
+		$instance = new GenericRegExTokenizer();
+
+		$this->assertEquals(
+			$expected,
+			$instance->tokenize( $string )
+		);
+	}
+
+	public function testTokenizeWithEnabledExemptionList() {
+		$string = "It's a test string (that has no);";
+
+		$tokenizer = $this->getMockBuilder( '\Onoi\Tesa\Tokenizer\Tokenizer' )
+			->disableOriginalConstructor()
+			->getMockForAbstractClass();
+
+		$tokenizer->expects( $this->once() )
+			->method( 'setOption' );
+
+		$tokenizer->expects( $this->once() )
+			->method( 'tokenize' )
+			->with( $string )
+			->willReturn( [ $string ] );
+
+		$instance = new GenericRegExTokenizer( $tokenizer );
+
+		$instance->setOption(
+			GenericRegExTokenizer::REGEX_EXEMPTION,
+			[ '\(', '\)', "'", ';' ]
+		);
+
+		$this->assertEquals(
+			[ "It's", 'a', 'test', 'string', '(that', 'has', 'no);' ],
+			$instance->tokenize( $string )
+		);
+	}
+
+	public function stringProvider() {
+		$provider[] = [
+			"It's a test string (that has no);deep meaning except0",
+			[ 'It', 's', 'a', 'test', 'string', 'that', 'has', 'no', 'deep', 'meaning', 'except', '0' ]
+		];
+
+		$provider[] = [
+			"Привет, мир! Меня зовут д'Артаньян %) цуацуа123123",
+			[ 'Привет', 'мир', 'Меня', 'зовут', 'д', "Артаньян", 'цуацуа', '123123' ]
+		];
+
+		$provider[] = [
+			"[[Действует на возбудителей]] Brucella spp., Legionella pneumophila, Salmonella typhi,(за исключением остальных рифампицинов) не отмечено. ...
+
+Фармакокинетика[править | править вики-текст]
+Рифампицин хорошо всасывается из желудочно-кишечного тракта.",
+			[
+				'Действует', 'на', 'возбудителей', 'Brucella', 'spp', 'Legionella', 'pneumophila', 'Salmonella', 'typhi', 'за', 'исключением', 'остальных', 'рифампицинов',
+				'не', 'отмечено', 'Фармакокинетика', 'править', 'править', 'вики', 'текст', 'Рифампицин', 'хорошо', 'всасывается', 'из', 'желудочно', 'кишечного', 'тракта'
+			]
+		];
+
+		return $provider;
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/tests/phpunit/Unit/Tokenizer/IcuWordBoundaryTokenizerTest.php

@@ -0,0 +1,169 @@
+<?php
+
+namespace Onoi\Tesa\Tests;
+
+use Onoi\Tesa\Tokenizer\IcuWordBoundaryTokenizer;
+use PHPUnit\Framework\TestCase;
+
+/**
+ * @covers \Onoi\Tesa\Tokenizer\IcuWordBoundaryTokenizer
+ * @group onoi-tesa
+ *
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class IcuWordBoundaryTokenizerTest extends TestCase {
+
+	protected function setUp(): void {
+		$instance = new IcuWordBoundaryTokenizer();
+
+		if ( !$instance->isAvailable() ) {
+			$this->markTestSkipped( 'ICU extension is not available.' );
+		}
+	}
+
+	public function testCanConstruct() {
+		$this->assertInstanceOf(
+			'\Onoi\Tesa\Tokenizer\IcuWordBoundaryTokenizer',
+			new IcuWordBoundaryTokenizer()
+		);
+	}
+
+	/**
+	 * @dataProvider stringProvider
+	 */
+	public function testTokenize( $string, $expected ) {
+		$instance = new IcuWordBoundaryTokenizer();
+
+		$this->assertEquals(
+			$expected,
+			$instance->tokenize( $string )
+		);
+	}
+
+	public function testSetOption() {
+		$tokenizer = $this->getMockBuilder( '\Onoi\Tesa\Tokenizer\Tokenizer' )
+			->disableOriginalConstructor()
+			->getMockForAbstractClass();
+
+		$tokenizer->expects( $this->once() )
+			->method( 'setOption' );
+
+		$instance = new IcuWordBoundaryTokenizer(
+			$tokenizer
+		);
+
+		$instance->setOption(
+			IcuWordBoundaryTokenizer::REGEX_EXEMPTION,
+			[ 'Foo' ]
+		);
+	}
+
+	public function testGeneralSetters() {
+		$tokenizer = $this->getMockBuilder( '\Onoi\Tesa\Tokenizer\Tokenizer' )
+			->disableOriginalConstructor()
+			->getMockForAbstractClass();
+
+		$instance = new IcuWordBoundaryTokenizer(
+			$tokenizer
+		);
+
+		$instance->setLocale( 'en' );
+		$instance->setWordTokenizerAttribute( false );
+
+		$this->assertFalse(
+			$instance->isWordTokenizer()
+		);
+	}
+
+	public function stringProvider() {
+		$provider[] = [
+			"安全テスト",
+			[ '安全', 'テスト' ]
+		];
+
+		// Would expect 'すもも', 'も', 'もも', 'も', 'もも', 'の', 'うち', '。'
+		$provider[] = [
+			"すもももももももものうち。",
+			[ 'すもも', 'も', 'も', 'も', 'も', 'も', 'もの', 'うち', '。' ]
+		];
+
+		$provider[] = [
+			"李も桃も桃のうち。",
+			[ '李', 'も', '桃', 'も', '桃', 'の', 'うち', '。' ]
+		];
+
+		$provider[] = [
+			"إسرائيل",
+			[ 'إسرائيل' ]
+		];
+
+		$provider[] = [
+			"검색엔ㅇㅏ진",
+			[ '검색엔', 'ㅇㅏ', '진' ]
+		];
+
+		$provider[] = [
+			"검색엔ㅇㅏ진1234abcdfrA",
+			[ '검색엔', 'ㅇㅏ', '진', '1234abcdfrA' ]
+		];
+
+		$provider[] = [
+			"1234abcdfrA",
+			[ '1234abcdfrA' ]
+		];
+
+		$provider[] = [
+			"公明執ようなＳＮＳもストーカー行為の対象に",
+			[
+				'公明', '執よう', 'な', 'ＳＮＳ', 'も',
+				'ストーカー', '行為', 'の', '対象', 'に'
+			]
+		];
+
+		$provider[] = [
+			"公明執",
+			[ '公明', '執' ]
+		];
+
+		$provider[] = [
+			"IQテスト",
+			[ 'IQ', 'テスト' ]
+		];
+
+		$provider[] = [
+			"foo テスト bar",
+			[ 'foo', 'テスト', 'bar' ]
+		];
+
+		$provider[] = [
+			"foo テスト bar 123abc ^&'",
+			[ 'foo', 'テスト', 'bar', '123abc', '^', '&', "'" ]
+		];
+
+		$provider[] = [
+			"was discovered in 1957 and first sold as a medication in 1971",
+			[
+				'was', 'discovered', 'in', '1957', 'and',
+				'first', 'sold', 'as', 'a', 'medication', 'in', '1971'
+			]
+		];
+
+		// See JaTinySegmenterTokenizerTest for comparison
+		$provider[] = [
+			'日本語の新聞記事であれば文字単位で95%程度の精度で分かち書きが行えます。 ',
+			[
+				'日本語', 'の', '新聞', '記事', 'で',
+				'あれ', 'ば', '文字', '単位',
+				'で', '95', '%', '程度',
+				'の', '精度', 'で', '分かち書き',
+				'が', '行', 'え', 'ます', '。'
+			]
+		];
+
+		return $provider;
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/tests/phpunit/Unit/Tokenizer/JaCompoundGroupTokenizerTest.php

@@ -0,0 +1,75 @@
+<?php
+
+namespace Onoi\Tesa\Tests;
+
+use Onoi\Tesa\Tokenizer\JaCompoundGroupTokenizer;
+use PHPUnit\Framework\TestCase;
+
+/**
+ * @covers \Onoi\Tesa\Tokenizer\JaCompoundGroupTokenizer
+ * @group onoi-tesa
+ *
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class JaCompoundGroupTokenizerTest extends TestCase {
+
+	public function testUnknownOption() {
+		$this->assertInstanceOf(
+			'\Onoi\Tesa\Tokenizer\JaCompoundGroupTokenizer',
+			new JaCompoundGroupTokenizer()
+		);
+	}
+
+	/**
+	 * @dataProvider stringProvider
+	 */
+	public function testTokenize( $string, $expected ) {
+		$instance = new JaCompoundGroupTokenizer();
+
+		$this->assertEquals(
+			$expected,
+			$instance->tokenize( $string )
+		);
+	}
+
+	public function testTokenizeWithOption() {
+		$string = 'と歓声を上げていました';
+
+		$tokenizer = $this->getMockBuilder( '\Onoi\Tesa\Tokenizer\Tokenizer' )
+			->disableOriginalConstructor()
+			->getMockForAbstractClass();
+
+		$tokenizer->expects( $this->once() )
+			->method( 'setOption' );
+
+		$tokenizer->expects( $this->once() )
+			->method( 'tokenize' )
+			->with( $string )
+			->willReturn( [ $string ] );
+
+		$instance = new JaCompoundGroupTokenizer( $tokenizer );
+
+		$instance->setOption(
+			JaCompoundGroupTokenizer::REGEX_EXEMPTION,
+			[ 'Foo' ]
+		);
+
+		$this->assertEquals(
+			[ '歓声', '上' ],
+			$instance->tokenize( $string )
+		);
+	}
+
+	public function stringProvider() {
+		$provider[] = [
+			'と歓声を上げていました。 十勝農業改良普及センターによりますと',
+			[ '歓声', '上', '十勝農業改良普及', 'センター' ]
+		];
+
+		return $provider;
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/tests/phpunit/Unit/Tokenizer/JaTinySegmenterTokenizerTest.php

@@ -0,0 +1,189 @@
+<?php
+
+namespace Onoi\Tesa\Tests;
+
+use Onoi\Tesa\Tokenizer\JaTinySegmenterTokenizer;
+use PHPUnit\Framework\TestCase;
+
+/**
+ * @covers \Onoi\Tesa\Tokenizer\JaTinySegmenterTokenizer
+ * @group onoi-tesa
+ *
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class JaTinySegmenterTokenizerTest extends TestCase {
+
+	public function testCanConstruct() {
+		$tokenizer = $this->getMockBuilder( '\Onoi\Tesa\Tokenizer\Tokenizer' )
+			->disableOriginalConstructor()
+			->getMockForAbstractClass();
+
+		$this->assertInstanceOf(
+			'\Onoi\Tesa\Tokenizer\JaTinySegmenterTokenizer',
+			new JaTinySegmenterTokenizer( $tokenizer )
+		);
+	}
+
+	/**
+	 * @dataProvider stringProvider
+	 */
+	public function testTokenize( $string, $expected ) {
+		$instance = new JaTinySegmenterTokenizer();
+
+		$this->assertEquals(
+			$expected,
+			$instance->tokenize( $string )
+		);
+	}
+
+	public function stringProvider() {
+		$provider[] = [
+			'極めてコンパクトな日本語分かち書きソフトウェアです。',
+			[
+				'極め', // should be 極めて
+				'て',
+				'コンパクト',
+				'な',
+				'日本',
+				'語分',
+				'かち',
+				'書き',
+				'ソフトウェア',
+				'です',
+				'。'
+			]
+		];
+
+		$provider[] = [
+			'日本語の新聞記事であれば文字単位で95%程度の精度で分かち書きが行えます。 ',
+			[
+				'日本語',
+				'の',
+				'新聞',
+				'記事',
+				'で',
+				'あれ',
+				'ば',
+				'文字',
+				'単位',
+				'で',
+				'9',
+				'5',
+				'%',
+				'程度',
+				'の',
+				'精度',
+				'で',
+				'分かち',
+				'書き',
+				'が',
+				'行え',
+				'ます',
+				'。'
+
+			]
+		];
+
+		$provider[] = [
+			'私の名前は中野です',
+			[
+				'私',
+				'の',
+				'名前',
+				'は',
+				'中野',
+				'です'
+			]
+		];
+
+		$provider[] = [
+			'TinySegmenterは25kBで書かれています。',
+			[
+				'TinySegmenter',
+				'は',
+				'2',
+				'5',
+				'kB',
+				'で',
+				'書か',
+				'れ',
+				'て',
+				'い',
+				'ます',
+				'。'
+			]
+		];
+
+		$provider[] = [
+			'隣の客はAK47振りかざしてギャアギャアわめきたてる客だ。',
+			[
+				'隣',
+				'の',
+				'客',
+				'は',
+				'AK',
+				'4',
+				'7',
+				'振り',
+				'かざ', // should be かざし
+				'し',
+				'て',
+				'ギャアギャア',
+				'わめき',
+				'た',
+				'てる',
+				'客',
+				'だ',
+				'。'
+			]
+		];
+
+		// See JaCompoundGroupTokenizerTest for comparison
+		$provider[] = [
+			'と歓声を上げていました。 十勝農業改良普及センターによりますと',
+			[
+				'と',
+				'歓声',
+				'を',
+				'上げ',
+				'て',
+				'い',
+				'まし',
+				'た',
+				'。',
+				'十勝農業',
+				'改良',
+				'普及',
+				'センター',
+				'により',
+				'ます',
+				'と'
+			]
+		];
+
+		// See IcuWordBoundaryTokenizerTest
+		$provider[] = [
+			"公明執ようなＳＮＳもストーカー行為の対象に",
+			[
+				'公明執',
+				'よう',
+				'な',
+				'ＳＮＳ',
+				'も',
+				'ストーカー',
+				'行為',
+				'の',
+				'対象',
+				'に'
+			]
+		];
+
+		// https://github.com/chezou/TinySegmenter.jl/blob/master/test/timemachineu8j.txt
+
+		return $provider;
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/tests/phpunit/Unit/Tokenizer/NGramTokenizerTest.php

@@ -0,0 +1,214 @@
+<?php
+
+namespace Onoi\Tesa\Tests;
+
+use Onoi\Tesa\Tokenizer\NGramTokenizer;
+use PHPUnit\Framework\TestCase;
+
+/**
+ * @covers \Onoi\Tesa\Tokenizer\NGramTokenizer
+ * @group onoi-tesa
+ *
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class NGramTokenizerTest extends TestCase {
+
+	public function testCanConstruct() {
+		$this->assertInstanceOf(
+			'\Onoi\Tesa\Tokenizer\NGramTokenizer',
+			new NGramTokenizer()
+		);
+	}
+
+	/**
+	 * @dataProvider stringProvider
+	 */
+	public function testTokenize( $string, $ngram, $expected ) {
+		$instance = new NGramTokenizer( null, $ngram );
+
+		$this->assertEquals(
+			$expected,
+			$instance->tokenize( $string )
+		);
+
+		$this->assertFalse(
+			$instance->isWordTokenizer()
+		);
+	}
+
+	public function testTokenizeWithStartEndMarker() {
+		// http://cloudmark.github.io/Language-Detection
+		$string = 'TEXT';
+
+		$expected = [
+			'_tex',
+			'text',
+			'ext_',
+			'xt__',
+			't___'
+		];
+
+		$instance = new NGramTokenizer( null, 4 );
+		$instance->withMarker( true );
+
+		$this->assertEquals(
+			$expected,
+			$instance->tokenize( $string )
+		);
+	}
+
+	public function testTokenizeWithStartEndMarker2() {
+		$string = '教授は';
+
+		$expected = [
+			'_教授',
+			'教授は',
+			'授は_',
+			'は__'
+		];
+
+		$instance = new NGramTokenizer( null, 3 );
+		$instance->withMarker( true );
+
+		$this->assertEquals(
+			$expected,
+			$instance->tokenize( $string )
+		);
+	}
+
+	public function testTokenizeWithOption() {
+		$string = '红色中华';
+
+		$tokenizer = $this->getMockBuilder( '\Onoi\Tesa\Tokenizer\Tokenizer' )
+			->disableOriginalConstructor()
+			->getMockForAbstractClass();
+
+		$tokenizer->expects( $this->once() )
+			->method( 'setOption' );
+
+		$tokenizer->expects( $this->once() )
+			->method( 'tokenize' )
+			->with( $string )
+			->willReturn( [ $string ] );
+
+		$instance = new NGramTokenizer( $tokenizer );
+
+		$instance->setOption(
+			NGramTokenizer::REGEX_EXEMPTION,
+			[ 'Foo' ]
+		);
+
+		$this->assertEquals(
+			[ '红色', '色中', '中华' ],
+			$instance->tokenize( $string )
+		);
+	}
+
+	public function stringProvider() {
+		$provider[] = [
+			'TEXT',
+			'4',
+			[
+				'text'
+			]
+		];
+
+		$provider[] = [
+			'12345678',
+			'2',
+			[
+				'12',
+				'23',
+				'34',
+				'45',
+				'56',
+				'67',
+				'78'
+			]
+		];
+
+		$provider[] = [
+			'12345678',
+			'3',
+			[
+				'123',
+				'234',
+				'345',
+				'456',
+				'567',
+				'678'
+			]
+		];
+
+		$provider[] = [
+			'hello',
+			'3',
+			[
+				'hel',
+				'ell',
+				'llo'
+			]
+		];
+
+		$provider[] = [
+			'Hello World!',
+			'3',
+			[
+				'hel',
+				'ell',
+				'llo',
+				'lo ',
+				'o w',
+				' wo',
+				'wor',
+				'orl',
+				'rld',
+				'ld!'
+			]
+		];
+
+		$provider[] = [
+			'Новости',
+			'3',
+			[
+				'нов',
+				'ово',
+				'вос',
+				'ост',
+				'сти'
+			]
+		];
+
+		$provider[] = [
+			'1時36分更新',
+			'3',
+			[
+				'1時3',
+				'時36',
+				'36分',
+				'6分更',
+				'分更新'
+			]
+		];
+
+		$provider[] = [
+			'こんにちは世界！',
+			'2',
+			[
+				'こん',
+				'んに',
+				'にち',
+				'ちは',
+				'は世',
+				'世界',
+				'界！'
+			]
+		];
+
+		return $provider;
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/tests/phpunit/Unit/Tokenizer/PunctuationRegExTokenizerTest.php

@@ -0,0 +1,113 @@
+<?php
+
+namespace Onoi\Tesa\Tests;
+
+use Onoi\Tesa\Tokenizer\PunctuationRegExTokenizer;
+use PHPUnit\Framework\TestCase;
+
+/**
+ * @covers \Onoi\Tesa\Tokenizer\PunctuationRegExTokenizer
+ * @group onoi-tesa
+ *
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class PunctuationRegExTokenizerTest extends TestCase {
+
+	public function testCanConstruct() {
+		$this->assertInstanceOf(
+			'\Onoi\Tesa\Tokenizer\PunctuationRegExTokenizer',
+			new PunctuationRegExTokenizer()
+		);
+	}
+
+	/**
+	 * @dataProvider stringProvider
+	 */
+	public function testTokenize( $string, $patternExemption, $expected ) {
+		$instance = new PunctuationRegExTokenizer();
+
+		$instance->setOption(
+			PunctuationRegExTokenizer::REGEX_EXEMPTION,
+			$patternExemption
+		);
+
+		$this->assertEquals(
+			$expected,
+			$instance->tokenize( $string )
+		);
+
+		$this->assertTrue(
+			$instance->isWordTokenizer()
+		);
+	}
+
+	public function testisWordTokenizerFromInheritTokenizer() {
+		$tokenizer = $this->getMockBuilder( '\Onoi\Tesa\Tokenizer\Tokenizer' )
+			->disableOriginalConstructor()
+			->getMockForAbstractClass();
+
+		$tokenizer->expects( $this->once() )
+			->method( 'isWordTokenizer' )
+			->willReturn( false );
+
+		$instance = new PunctuationRegExTokenizer( $tokenizer );
+
+		$this->assertFalse(
+			$instance->isWordTokenizer()
+		);
+	}
+
+	public function testTokenizeWithOption() {
+		$string = '123, 345';
+
+		$tokenizer = $this->getMockBuilder( '\Onoi\Tesa\Tokenizer\Tokenizer' )
+			->disableOriginalConstructor()
+			->getMockForAbstractClass();
+
+		$tokenizer->expects( $this->once() )
+			->method( 'setOption' );
+
+		$tokenizer->expects( $this->once() )
+			->method( 'tokenize' )
+			->with( $string )
+			->willReturn( [ $string ] );
+
+		$instance = new PunctuationRegExTokenizer( $tokenizer );
+
+		$instance->setOption(
+			PunctuationRegExTokenizer::REGEX_EXEMPTION,
+			[ ',' ]
+		);
+
+		$this->assertEquals(
+			[ '123,', '345' ],
+			$instance->tokenize( $string )
+		);
+	}
+
+	public function stringProvider() {
+		$provider[] = [
+			'123, 345^456&[foo:bar]',
+			'',
+			[ '123', '345', '456', 'foo', 'bar' ]
+		];
+
+		$provider[] = [
+			'123, 345^456&[foo:bar]',
+			[ ',', '&' ],
+			[ '123,', '345', '456&', 'foo', 'bar' ]
+		];
+
+		$provider[] = [
+			'123, 345^456&[foo:bar] 3.',
+			[ ',', '&' ],
+			[ '123,', '345', '456&', 'foo', 'bar', '3' ]
+		];
+
+		return $provider;
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/Tesa/tests/phpunit/Unit/TransliteratorTest.php

@@ -0,0 +1,160 @@
+<?php
+
+namespace Onoi\Tesa\Tests;
+
+use Onoi\Tesa\Transliterator;
+use PHPUnit\Framework\TestCase;
+
+/**
+ * @covers \Onoi\Tesa\Transliterator
+ * @group onoi-tesa
+ *
+ * @license GPL-2.0-or-later
+ * @since 0.1
+ *
+ * @author mwjames
+ */
+class TransliteratorTest extends TestCase {
+
+	/**
+	 * @dataProvider characterProvider
+	 */
+	public function testTransliteration( $input, $flag, $expected ) {
+		$this->assertEquals(
+			$expected,
+			Transliterator::transliterate( $input, $flag )
+		);
+	}
+
+	public function testTransliterationWithoutOptionFlag() {
+		$this->assertEquals(
+			'aaaaaea',
+			Transliterator::transliterate( 'àáâãäå' )
+		);
+	}
+
+	public function characterProvider() {
+		$provider[] = [
+			'Foo',
+			'unknownFlag',
+			'Foo',
+		];
+
+		$provider[] = [
+			'ÀÁÂÃÄÅàáâãäåÒÓÔÕÕÖØòóôõöøÈÉÊËèéêëðÇçÐÌÍÎÏìíîïÙÚÛÜùúûüÑñŠšŸÿýŽž',
+			Transliterator::NONE,
+			'ÀÁÂÃÄÅàáâãäåÒÓÔÕÕÖØòóôõöøÈÉÊËèéêëðÇçÐÌÍÎÏìíîïÙÚÛÜùúûüÑñŠšŸÿýŽž',
+		];
+
+		$provider[] = [
+			'ÀÁÂÃÄÅàáâãäåÒÓÔÕÕÖØòóôõöøÈÉÊËèéêëðÇçÐÌÍÎÏìíîïÙÚÛÜùúûüÑñŠšŸÿýŽž',
+			Transliterator::DIACRITICS,
+			'AAAAAEAaaaaaeaOOOOOOEOoooooeoEEEEeeeeðCcÐIIIIiiiiUUUUEuuuueNnSsYyyZz'
+		];
+
+		$provider[] = [
+			'ỆᶍǍᶆṔƚÉ áéíóúýčďěňřšťžů',
+			Transliterator::DIACRITICS,
+			'ExAmPlE aeiouycdenrstzu'
+		];
+
+		$provider[] = [
+			'àáâãäå',
+			Transliterator::DIACRITICS,
+			'aaaaaea'
+		];
+
+		$provider[] = [
+			'èéêë',
+			Transliterator::DIACRITICS,
+			'eeee'
+		];
+
+		$provider[] = [
+			'òóôõö',
+			Transliterator::DIACRITICS,
+			'oooooe'
+		];
+
+		$provider[] = [
+			'ùúûü',
+			Transliterator::DIACRITICS,
+			'uuuue'
+		];
+
+		$provider[] = [
+			'ç',
+			Transliterator::DIACRITICS,
+			'c'
+		];
+
+		$provider[] = [
+			'æ',
+			Transliterator::DIACRITICS,
+			'ae'
+		];
+
+		$provider[] = [
+			'ñ',
+			Transliterator::DIACRITICS,
+			'n'
+		];
+
+		$provider[] = [
+			'œ',
+			Transliterator::DIACRITICS,
+			'oe'
+		];
+
+		$provider[] = [
+			'ýÿ',
+			Transliterator::DIACRITICS,
+			'yy'
+		];
+
+		$provider[] = [
+			'ß',
+			Transliterator::DIACRITICS,
+			'ss'
+		];
+
+		$provider[] = [
+			'Vilʹândimaa',
+			Transliterator::DIACRITICS,
+			'Vilʹandimaa'
+		];
+
+		$provider[] = [
+			'Ελληνική Δημοκρατία',
+			Transliterator::GREEK,
+			'Ellīnikī́ Dīmokratía'
+		];
+
+		$provider[] = [
+			'Ελληνική Δημοκρατία',
+			Transliterator::DIACRITICS | Transliterator::GREEK,
+			'Ellinikí Dimokratia'
+		];
+
+		$provider[] = [
+			'Γκ γκ γξ Ει ει Ηυ Μπ μπ',
+			Transliterator::GREEK,
+			'Gk gk gx Ei ei Īy Mp mp'
+		];
+
+		$provider[] = [
+			'Μετατροπή του ελληνικού αλφαβήτου με λατινικούς χαρακτήρες',
+			Transliterator::GREEK,
+			'Metatropī́ tou ellīnikoú alfavī́tou me latinikoús charaktī́res',
+		];
+
+		$provider[] = [
+			'Ελληνικός Οργανισμός Τυποποίησης',
+			Transliterator::GREEK,
+			'Ellīnikós Organismós Typopoíīsīs',
+		];
+
+		return $provider;
+	}
+
+}

mediawiki/extensions/SemanticMediaWiki/codecov.yml

@@ -0,0 +1,2 @@
+fixes:
+  - "/var/www/html/extensions/SemanticMediaWiki/::"

mediawiki/extensions/SemanticMediaWiki/composer.json

@@ -0,0 +1,137 @@
+{
+	"name": "mediawiki/semantic-media-wiki",
+	"type": "mediawiki-extension",
+	"description": "An extension to MediaWiki that lets you store and query structured data within wiki pages",
+	"keywords": [
+		"SMW",
+		"Semantic MediaWiki",
+		"Wiki",
+		"MediaWiki"
+	],
+	"homepage": "https://www.semantic-mediawiki.org/wiki/",
+	"license": "GPL-2.0-or-later",
+	"authors": [
+		{
+			"name": "Markus Krötzsch",
+			"homepage": "https://www.korrekt.org/",
+			"role": "Original author"
+		},
+		{
+			"name": "Jeroen De Dauw",
+			"email": "jeroendedauw@gmail.com",
+			"homepage": "https://www.entropywins.wtf/",
+			"role": "Core developer"
+		},
+		{
+			"name": "James Hong Kong",
+			"homepage": "https://www.semantic-mediawiki.org/wiki/User:MWJames",
+			"role": "Core developer"
+		},
+		{
+			"name": "Professional Wiki",
+			"email": "info@professional.wiki",
+			"homepage": "https://professional.wiki",
+			"role": "Developer"
+		}
+	],
+	"support": {
+		"email": "semediawiki-user@lists.sourceforge.net",
+		"issues": "https://github.com/SemanticMediaWiki/SemanticMediaWiki/issues",
+		"forum": "https://www.semantic-mediawiki.org/wiki/semantic-mediawiki.org_talk:Community_portal",
+		"wiki": "https://www.semantic-mediawiki.org/wiki/",
+		"source": "https://github.com/SemanticMediaWiki/SemanticMediaWiki"
+	},
+	"require": {
+		"php": ">=8.1",
+		"ext-mbstring": "*",
+		"composer/installers": "^2.2.0|^1.0.1",
+		"psr/log": "~1.0",
+		"mediawiki/parser-hooks": "~1.4",
+		"param-processor/param-processor": "~1.2",
+		"serialization/serialization": "~3.2|~4.0",
+		"onoi/message-reporter": "~1.0",
+		"onoi/cache": "~1.2",
+		"onoi/event-dispatcher": "~1.0",
+		"onoi/blob-store": "~1.2",
+		"mediawiki/http-request": "~2.0|~1.1",
+		"onoi/callback-container": "~2.0",
+		"symfony/css-selector": "^5|^4|^3.3",
+		"seld/jsonlint": "^1.7",
+		"justinrainbow/json-schema": "~5.2",
+		"jeroen/file-fetcher": "^6|^5|^4.4",
+		"wikimedia/cdb": "^3|^2|^1",
+		"wikimedia/textcat": "^2|^1.1"
+	},
+	"require-dev": {
+		"mediawiki/mediawiki-codesniffer": "46.0.0",
+		"mediawiki/mediawiki-phan-config": "0.14.0",
+		"mediawiki/minus-x": "1.1.3",
+		"phpmd/phpmd": "~2.1",
+		"php-parallel-lint/php-console-highlighter": "1.0.0",
+		"php-parallel-lint/php-parallel-lint": "1.4.0",
+		"squizlabs/php_codesniffer": "~3.5"
+	},
+	"suggest": {
+		"mediawiki/semantic-result-formats": "Provides additional result formats for queries of structured data"
+	},
+	"extra": {
+		"branch-alias": {
+			"dev-master": "6.x-dev"
+		}
+	},
+	"autoload": {
+		"psr-4": {
+			"SMW\\": "src/",
+			"SMW\\Maintenance\\": "maintenance/",
+			"SMW\\Tests\\": "tests/phpunit/",
+			"Onoi\\Tesa\\": "Tesa/src/"
+		},
+		"psr-0": {
+			"SemanticMediaWiki": "includes/SemanticMediaWiki.php"
+		},
+		"files": [
+			"includes/GlobalFunctions.php"
+		],
+		"classmap" : [
+			"includes/",
+			"tests/phpunit/includes/"
+		]
+	},
+	"config": {
+		"process-timeout": 0,
+		"allow-plugins": {
+			"composer/installers": true,
+			"dealerdirect/phpcodesniffer-composer-installer": true
+		}
+	},
+	"scripts": {
+		"test": [
+			"@lint",
+			"@phpcs",
+			"minus-x check .",
+			"@phpunit:unit",
+			"@phpunit:integration"
+		],
+		"test-coverage": [
+			"@phpunit-coverage"
+		],
+		"analyze": [
+			"@lint",
+			"@phpcs"
+		],
+		"fix": [
+			"@phpcs-fix"
+		],
+		"lint": "parallel-lint . --exclude vendor --exclude node_modules",
+		"phpcs": "phpcs -sp --cache --parallel=$(nproc || sysctl -n hw.logicalcpu || powershell \"Get-WmiObject Win32_Processor | Select-Object -ExpandProperty NumberOfCores .trim()\" || echo 4 )",
+		"phpcs-fix": "phpcbf",
+		"phpunit": "php ${MW_INSTALL_PATH:-../..}/tests/phpunit/phpunit.php -c phpunit.xml.dist --bootstrap tests/bootstrap.php",
+		"phpunit:unit": "@phpunit --testsuite=semantic-mediawiki-unit",
+		"phpunit:integration": "@phpunit --testsuite semantic-mediawiki-check,semantic-mediawiki-data-model,semantic-mediawiki-integration,semantic-mediawiki-import,semantic-mediawiki-structure",
+		"phpunit-coverage": "@phpunit --testdox --coverage-text --coverage-clover coverage/php/coverage.xml",
+		"phpdbg": "phpdbg -qrr ${MW_INSTALL_PATH}/tests/phpunit/phpunit.php -c phpunit.xml.dist",
+		"minus-x": "minus-x check .",
+		"benchmark": "composer phpunit  -- --group semantic-mediawiki-benchmark",
+		"quick-benchmark": "php ${MW_INSTALL_PATH}/tests/phpunit/phpunit.php -c ./tests/phpunit/Benchmark/phpunit.quick.xml.dist --group semantic-mediawiki-benchmark"
+	}
+}

mediawiki/extensions/SemanticMediaWiki/data/config/db-primary-keys.php

@@ -0,0 +1,151 @@
+<?php
+
+use MediaWiki\MediaWikiServices;
+use Onoi\MessageReporter\MessageReporter;
+use SMW\Utils\CliMsgFormatter;
+
+/**
+ * This profile adds primary keys to Semantic MediaWiki owned tables to help DB
+ * environments like `Percona XtraDB Cluster` (#4724, #4507) that require those
+ * keys.
+ *
+ * Beware that while the profile is provided as part of the Semantic MediaWiki
+ * software it does NOT imply that systems like `Percona` are officially
+ * supported as there is no test setup that runs the required test suite.
+ *
+ * @see https://www.semantic-mediawiki.org/wiki/Config_preloading
+ *
+ * @since 3.2
+ */
+
+class ConfigPreloadPrimaryKeyTableMutator {
+
+	// #3559
+	const PRIMARY_KEYS = [
+
+		// Common property value tables
+		'smw_di_blob'     => 's_id,p_id,o_hash',
+		'smw_di_bool'     => 's_id,p_id,o_value',
+		'smw_di_coords'   => 's_id,p_id,o_serialized',
+		'smw_di_number'   => 's_id,p_id,o_serialized',
+		'smw_di_time'     => 's_id,p_id,o_serialized',
+		'smw_di_uri'      => 's_id,p_id,o_serialized',
+		'smw_di_wikipage' => 's_id,p_id,o_id',
+
+		// Fixed property value tables
+		'smw_fpt_ask'     => 's_id,o_id',
+		'smw_fpt_askde'   => 's_id,o_serialized',
+		'smw_fpt_askfo'   => 's_id,o_hash',
+		'smw_fpt_askdu'   => 's_id,o_serialized',
+		'smw_fpt_asksi'   => 's_id,o_serialized',
+		'smw_fpt_askst'   => 's_id,o_hash',
+		'smw_fpt_askpa'   => 's_id,o_hash',
+		'smw_fpt_cdat'    => 's_id,o_serialized',
+		'smw_fpt_conc'    => 's_id',
+		'smw_fpt_conv'    => 's_id,o_hash',
+		'smw_fpt_dtitle'  => 's_id,o_hash',
+		'smw_fpt_impo'    => 's_id,o_hash',
+		'smw_fpt_inst'    => 's_id,o_id',
+		'smw_fpt_lcode'   => 's_id,o_hash',
+		'smw_fpt_ledt'    => 's_id,o_id',
+		'smw_fpt_list'    => 's_id,o_hash',
+		'smw_fpt_mdat'    => 's_id,o_serialized',
+		'smw_fpt_media'   => 's_id,o_hash',
+		'smw_fpt_mime'    => 's_id,o_hash',
+		'smw_fpt_newp'    => 's_id,o_value',
+		'smw_fpt_pplb'    => 's_id,o_id',
+		'smw_fpt_prec'    => 's_id,o_serialized',
+		'smw_fpt_pval'    => 's_id,o_hash',
+		'smw_fpt_redi'    => 's_title,s_namespace',
+		'smw_fpt_serv'    => 's_id,o_hash',
+		'smw_fpt_sobj'    => 's_id,o_id',
+		'smw_fpt_subc'    => 's_id,o_id',
+		'smw_fpt_subp'    => 's_id,o_id',
+		'smw_fpt_text'    => 's_id,o_hash',
+		'smw_fpt_type'    => 's_id,o_serialized',
+		'smw_fpt_unit'    => 's_id,o_hash',
+		'smw_fpt_uri'     => 's_id,o_serialized',
+
+		// Other data tables
+		'smw_object_ids'  => 'smw_id',
+		'smw_object_aux'  => 'smw_id',
+		'smw_prop_stats'  => 'p_id',
+		'smw_query_links' => 's_id,o_id',
+		'smw_ft_search'   => 's_id,p_id',
+		'smw_concept_cache' => 's_id,o_id'
+	];
+
+	/**
+	 * @param string $tableName
+	 */
+	public function hasKey( string $tableName ): bool {
+		return self::PRIMARY_KEYS[$tableName] ?? false;
+	}
+
+	/**
+	 * @param string $tableName
+	 */
+	public function getKey( string $tableName ): string {
+		return self::PRIMARY_KEYS[$tableName];
+	}
+}
+
+/**
+ * @see https://github.com/SemanticMediaWiki/SemanticMediaWiki/blob/master/docs/examples/hook.sqlstore.installer.beforecreatetablescomplete.md
+ */
+MediaWikiServices::getInstance()->getHookContainer()->register( 'SMW::SQLStore::Installer::BeforeCreateTablesComplete', static function ( array $tables, MessageReporter $messageReporter ) {
+	$cliMsgFormatter = new CliMsgFormatter();
+	$configPreloadPrimaryKeyTableMutator = new ConfigPreloadPrimaryKeyTableMutator();
+
+	$messageReporter->reportMessage(
+		$cliMsgFormatter->section( 'Primary key(s)', 3, '-', true )
+	);
+
+	$i = 0;
+
+	$text = [
+		'The following updates adds primary key information for the tables',
+		'owned by Semantic MediaWiki.'
+	];
+
+	$messageReporter->reportMessage(
+		"\n" . $cliMsgFormatter->wordwrap( $text ) . "\n"
+	);
+
+	$messageReporter->reportMessage(
+		"\n" . $cliMsgFormatter->oneCol( "Checking table definitions ..." )
+	);
+
+	/**
+	 * @var \SMW\SQLStore\TableBuilder\Table[]
+	 */
+	foreach ( $tables as $table ) {
+
+		$tableName = $table->getName();
+
+		if ( !$configPreloadPrimaryKeyTableMutator->hasKey( $tableName ) ) {
+			continue;
+		}
+
+		$i++;
+
+		$table->setPrimaryKey(
+			$configPreloadPrimaryKeyTableMutator->getKey( $tableName )
+		);
+	}
+
+	$messageReporter->reportMessage(
+		$cliMsgFormatter->twoCols( "... run table definition update ...", "$i (tables)", 3 )
+	);
+
+	$messageReporter->reportMessage(
+		$cliMsgFormatter->oneCol( "... done.", 3 )
+	);
+} );
+
+return [
+
+	// Modify the upgrade key to make sure an update is forced in the event this
+	// profile is used (or removed).
+	'smwgUpgradeKey' => $GLOBALS['smwgUpgradeKey'] . ':primary'
+];

mediawiki/extensions/SemanticMediaWiki/data/config/developer.php

@@ -0,0 +1,62 @@
+<?php
+
+/**
+ * Default settings mostly used in connection with a development environment and
+ * it not meant to be used in production.
+ *
+ * @since 3.2
+ */
+
+error_reporting( E_STRICT | E_ALL );
+ini_set( "display_errors", 1 );
+
+return [
+
+	/**
+	 *  MediaWiki specific settings to be used within a development environment
+	 */
+	'wgShowExceptionDetails' => true,
+	'wgDevelopmentWarnings' => true,
+	'wgShowSQLErrors' => true,
+	'wgDebugDumpSql' => true,
+	'wgShowDBErrorBacktrace' => true,
+
+	/**
+	 * @see https://www.mediawiki.org/wiki/Debugging_toolbar
+	 *
+	 * A utility for developers that displays debug information about a MediaWiki
+	 * page at the bottom of the browser window.
+	 */
+	'wgDebugToolbar' => true,
+
+	/**
+	 * Semantic MediaWiki related
+	 */
+
+	/**
+	 * @see $smwgIgnoreExtensionRegistrationCheck
+	 */
+	'smwgIgnoreExtensionRegistrationCheck' => true,
+
+	/**
+	 * @see $smwgDefaultLoggerRole
+	 *
+	 * You never want this role to be enabled in production because it will create
+	 * large log files while monitoring SMW related activities in detail.
+	 */
+	'smwgDefaultLoggerRole' => 'developer',
+
+	/**
+	 * @see $smwgJobQueueWatchlist
+	 */
+	'smwgJobQueueWatchlist' => [
+		'smw.update',
+		'smw.fulltextSearchTableUpdate',
+		'smw.changePropagationUpdate',
+		'smw.changePropagationClassUpdate',
+		'smw.changePropagationDispatch',
+		'smw.elasticIndexerRecovery',
+		'smw.elasticFileIngest'
+	]
+
+];

mediawiki/extensions/SemanticMediaWiki/data/config/elastic-fileingest.php

@@ -0,0 +1,35 @@
+<?php
+
+/**
+ * Convenience settings to extend Semantic MediaWiki and support file ingestion
+ * in connection with the ElasticStore.
+ *
+ * @since 3.2
+ */
+
+$extraSettings = [
+	'settings' => [
+		'data' => [ "index.mapping.total_fields.limit" => 12000 ]
+	],
+	'indexer' => [
+		"raw.text" => true,
+		"experimental.file.ingest" => true,
+		"throw.exception.on.illegal.argument.error" => false
+	],
+	"query" => [
+		"highlight.fragment" => [ "type" => "unified" ]
+	]
+];
+
+return [
+
+	/**
+	 * @see $smwgDefaultStore
+	 */
+	'smwgDefaultStore' => 'SMW\Elastic\ElasticStore',
+
+	/**
+	 * @see $smwgElasticsearchConfig
+	 */
+	'smwgElasticsearchConfig' => array_replace_recursive( $GLOBALS['smwgElasticsearchConfig'], $extraSettings ),
+];

mediawiki/extensions/SemanticMediaWiki/data/config/media.php

@@ -0,0 +1,17 @@
+<?php
+
+/**
+ * Convenience settings to extend Semantic MediaWiki with media related functionality.
+ *
+ * @since 3.2
+ */
+
+$properties = [ '_MIME', '_MEDIA', '_ATTCH_LINK' ];
+
+return [
+
+	/**
+	 * @see $smwgPageSpecialProperties
+	 */
+	'smwgPageSpecialProperties' => array_merge( $GLOBALS['smwgPageSpecialProperties'], $properties )
+];

mediawiki/extensions/SemanticMediaWiki/data/elastic/default-profile.json

@@ -0,0 +1,56 @@
+{
+	"connection": {
+		"retries": 2,
+		"timeout": 30,
+		"connect_timeout": 30,
+		"quick_ping": true
+	},
+	"settings": {
+		"data": {
+			"index.mapping.total_fields.limit": 9000,
+			"index.max_result_window": 50000
+		}
+	},
+	"indexer": {
+		"raw.text": false,
+		"experimental.file.ingest": false,
+		"throw.exception.on.illegal.argument.error": true,
+		"job.recovery.retries": 5,
+		"job.file.ingest.retries": 3,
+		"monitor.entity.replication": true,
+		"monitor.entity.replication.cache_lifetime": 3600,
+		"data.sqlstore_compatibility": true
+	},
+	"query": {
+		"fallback.no_connection": false,
+		"profiling": false,
+		"debug.explain": true,
+		"debug.description.log": true,
+		"maximum.value.length": 500,
+		"must_not.property.exists": true,
+		"sort.property.must.exists": true,
+		"score.sortfield": "es.score",
+		"query_string.boolean.operators": true,
+		"compat.mode": true,
+		"subquery.size":10000,
+		"subquery.constant.score": true,
+		"subquery.terms.lookup.result.size.index.write.threshold": 200,
+		"subquery.terms.lookup.cache.lifetime": 3600,
+		"concept.terms.lookup": true,
+		"concept.terms.lookup.result.size.index.write.threshold": 10,
+		"concept.terms.lookup.cache.lifetime": 3600,
+		"cjk.best.effort.proximity.match": true,
+		"wide.proximity.as.match_phrase": true,
+		"wide.proximity.fields": [
+			"subject.title^8",
+			"text_copy^5",
+			"text_raw",
+			"attachment.title^3",
+			"attachment.content"
+		],
+		"uri.field.case.insensitive": false,
+		"text.field.case.insensitive.eq.match": false,
+		"page.field.case.insensitive.proximity.match": true,
+		"highlight.fragment": { "number" : 1, "size" : 250, "type" : false }
+	}
+}

mediawiki/extensions/SemanticMediaWiki/data/elastic/smw-data-icu.json

@@ -0,0 +1,286 @@
+{
+	"settings": {
+		"number_of_shards": 2,
+		"number_of_replicas": 2,
+		"refresh_interval" : "1s",
+		"index.mapping.total_fields.limit": "9000",
+		"index.max_result_window": "50000",
+		"analysis": {
+			"filter": {
+				"uri_stopwords_filter": {
+					"type": "stop",
+					"stopwords": [ "http", "https", "ftp", "www" ]
+				}
+			},
+			"char_filter": {
+				"nfd_normalizer": {
+					"type": "icu_normalizer",
+					"name": "nfc",
+					"mode": "decompose"
+				},
+				"nfkc_normalizer": {
+					"type": "icu_normalizer",
+					"name": "nfkc",
+					"mode": "decompose"
+				},
+				"wiki_char_filter": {
+					"type": "mapping",
+					"mappings": [
+						"[ => ",
+						"] => ",
+						"/ => "
+					]
+				}
+			},
+			"analyzer": {
+				"uri_lowercase_with_stopwords": {
+					"type": "custom",
+					"tokenizer": "lowercase",
+					"filter": [ "uri_stopwords_filter" ]
+				},
+				"nfkc_cf_normalized": {
+					"tokenizer": "icu_tokenizer",
+					"char_filter": [
+						"icu_normalizer"
+					]
+				},
+				"nfkc_cf_normalized_lowercase": {
+					"tokenizer": "icu_tokenizer",
+					"char_filter": [
+						"icu_normalizer"
+					],
+					"filter": [ "lowercase" ]
+				},
+				"nfd_normalized": {
+					"tokenizer": "icu_tokenizer",
+					"char_filter": [
+						"nfd_normalizer"
+					]
+				},
+				"nfkc_normalized": {
+					"tokenizer": "icu_tokenizer",
+					"char_filter": [
+						"nfkc_normalizer"
+					]
+				}
+			}
+		}
+	},
+	"mappings": {
+		"data": {
+			"dynamic_templates": [
+				{
+					"text_fields": {
+						"path_match": "P:*.txtField",
+						"match_mapping_type": "*",
+						"mapping": {
+							"type": "text",
+							"copy_to": "text_copy",
+							"analyzer": "nfkc_cf_normalized",
+							"fields": {
+								"sort": {
+									"type": "icu_collation_keyword",
+									"index": false,
+									"ignore_above": 256
+								},
+								"keyword": {
+									"type": "keyword",
+									"ignore_above": 2000
+								}
+							}
+						}
+					}
+				},
+				{
+					"uri_fields": {
+						"path_match": "P:*.uriField",
+						"match_mapping_type": "*",
+						"mapping": {
+							"type": "text",
+							"copy_to": "text_copy",
+							"fields": {
+								"sort": {
+									"type": "icu_collation_keyword",
+									"index": false,
+									"ignore_above": 256
+								},
+								"keyword": {
+									"type": "keyword",
+									"ignore_above": 2000
+								},
+								"lowercase": {
+									"type":     "text",
+									"analyzer": "uri_lowercase_with_stopwords"
+								}
+							}
+						}
+					}
+				},
+				{
+					"page_fields_text": {
+						"path_match": "P:*.wpgField",
+						"match_mapping_type": "*",
+						"mapping": {
+							"type": "text",
+							"copy_to": "text_copy",
+							"analyzer": "nfkc_cf_normalized",
+							"fields": {
+								"sort": {
+									"type": "icu_collation_keyword",
+									"index": false,
+									"ignore_above": 256
+								},
+								"keyword": {
+									"type": "keyword",
+									"ignore_above": 500
+								},
+								"lowercase": {
+									"type": "keyword",
+									"analyzer": "nfkc_cf_normalized_lowercase"
+								}
+							}
+						}
+					}
+				},
+				{
+					"page_fields_identifier": {
+						"path_match": "P:*.wpgID",
+						"match_mapping_type": "*",
+						"mapping": {
+							"type": "long"
+						}
+					}
+				},
+				{
+					"numeric_fields": {
+						"path_match": "P:*.numField",
+						"match_mapping_type": "*",
+						"mapping": {
+							"type": "double",
+							"fields": {
+								"keyword": {
+									"type": "keyword",
+									"ignore_above": 256
+								}
+							}
+						}
+					}
+				},
+				{
+					"date_fields": {
+						"path_match": "P:*.datField",
+						"match_mapping_type": "*",
+						"mapping": {
+							"type": "double",
+							"fields": {
+								"keyword": {
+									"type": "keyword",
+									"ignore_above": 256
+								}
+							}
+						}
+					}
+				},
+				{
+					"date_fields_raw": {
+						"path_match": "P:*.dat_raw",
+						"match_mapping_type": "*",
+						"mapping": {
+							"type": "keyword"
+						}
+					}
+				},
+				{
+					"geo_fields": {
+						"path_match": "P:*.geoField",
+						"match_mapping_type": "string",
+						"mapping": {
+							"type": "keyword",
+							"fields": {
+								"point": {
+									"type": "geo_point"
+								}
+							}
+						}
+					}
+				},
+				{
+					"boolean_fields": {
+						"path_match": "P:*.booField",
+						"match_mapping_type": "boolean",
+						"mapping": {
+							"type": "boolean"
+						}
+					}
+				}
+			],
+			"properties": {
+				"noop": {
+					"type": "integer"
+				},
+				"text_copy": {
+					"type": "text",
+					"analyzer": "nfkc_cf_normalized",
+					"doc_values": false
+				},
+				"text_raw": {
+					"type": "text",
+					"analyzer": "nfkc_cf_normalized",
+					"doc_values": false
+				},
+				"subject.title": {
+					"type": "text",
+					"fields": {
+						"sort": {
+							"type": "icu_collation_keyword",
+							"index": false
+						},
+						"keyword": {
+							"type": "keyword",
+							"ignore_above": 256
+						}
+					}
+				},
+				"subject.interwiki": {
+					"type": "text",
+					"fields": {
+						"keyword": {
+							"type": "keyword",
+							"ignore_above": 256
+						}
+					}
+				},
+				"subject.subobject": {
+					"type": "text",
+					"fields": {
+						"keyword": {
+							"type": "keyword",
+							"ignore_above": 256
+						}
+					}
+				},
+				"subject.sortkey": {
+					"type": "text",
+					"copy_to": "text_copy",
+					"fields": {
+						"sort": {
+							"type": "icu_collation_keyword",
+							"index": false
+						},
+						"keyword": {
+							"type": "keyword",
+							"ignore_above": 256
+						},
+						"lowercase": {
+							"type": "keyword",
+							"analyzer": "nfkc_cf_normalized_lowercase"
+						}
+					}
+				},
+				"subject.rev_id": {
+					"type": "integer"
+				}
+			}
+		}
+	}
+}

mediawiki/extensions/SemanticMediaWiki/data/elastic/smw-data-standard.json

@@ -0,0 +1,273 @@
+{
+	"settings": {
+		"number_of_shards": 2,
+		"number_of_replicas": 2,
+		"refresh_interval" : "1s",
+		"index.mapping.total_fields.limit": "9000",
+		"index.max_result_window": "50000",
+		"analysis": {
+			"filter": {
+				"uri_stopwords_filter": {
+					"type": "stop",
+					"stopwords": [ "http", "https", "ftp", "www" ]
+				}
+			},
+			"char_filter": {
+				"wiki_char_filter": {
+					"type": "mapping",
+					"mappings": [
+						"[ => ",
+						"] => ",
+						"/ => "
+					]
+				}
+			},
+			"analyzer": {
+				"uri_lowercase_with_stopwords": {
+					"type": "custom",
+					"tokenizer": "lowercase",
+					"filter": [ "uri_stopwords_filter" ]
+				},
+				"exact_text_lowercase": {
+					"type": "custom",
+					"tokenizer": "keyword",
+					"filter": [ "lowercase", "asciifolding" ]
+				}
+			},
+			"normalizer": {
+				"standard_sort_normalizer": {
+					"type": "custom",
+					"char_filter": [ "wiki_char_filter" ]
+				},
+				"char_normalizer": {
+					"type": "custom",
+					"char_filter": [ "wiki_char_filter" ]
+				},
+				"lowercase_normalizer": {
+					"type": "custom",
+					"char_filter": [],
+					"filter": [ "lowercase" ]
+				},
+				"case_insensitive_sort_normalizer": {
+					"type": "custom",
+					"char_filter": [ "wiki_char_filter" ],
+					"filter": [ "lowercase", "asciifolding" ]
+				}
+			}
+		}
+	},
+	"mappings": {
+		"dynamic_templates": [
+			{
+				"text_fields": {
+					"path_match": "P:*.txtField",
+					"match_mapping_type": "*",
+					"mapping": {
+						"type": "text",
+						"copy_to": "text_copy",
+						"fields": {
+							"sort": {
+								"type": "keyword",
+								"normalizer": "standard_sort_normalizer",
+								"index": false,
+								"ignore_above": 256
+							},
+							"keyword": {
+								"type": "keyword",
+								"ignore_above": 2000
+							}
+						}
+					}
+				}
+			},
+			{
+				"uri_fields": {
+					"path_match": "P:*.uriField",
+					"match_mapping_type": "*",
+					"mapping": {
+						"type": "text",
+						"copy_to": "text_copy",
+						"fields": {
+							"sort": {
+								"type": "keyword",
+								"normalizer": "standard_sort_normalizer",
+								"index": false,
+								"ignore_above": 256
+							},
+							"keyword": {
+								"type": "keyword",
+								"ignore_above": 2000
+							},
+							"lowercase": {
+								"type":     "text",
+								"analyzer": "uri_lowercase_with_stopwords"
+							}
+						}
+					}
+				}
+			},
+			{
+				"page_fields_text": {
+					"path_match": "P:*.wpgField",
+					"match_mapping_type": "*",
+					"mapping": {
+						"type": "text",
+						"copy_to": "text_copy",
+						"fields": {
+							"sort": {
+								"type": "keyword",
+								"normalizer": "standard_sort_normalizer",
+								"index": false,
+								"ignore_above": 256
+							},
+							"keyword": {
+								"type": "keyword",
+								"ignore_above": 500
+							},
+							"lowercase": {
+								"type": "keyword",
+								"normalizer": "lowercase_normalizer"
+							}
+						}
+					}
+				}
+			},
+			{
+				"page_fields_identifier": {
+					"path_match": "P:*.wpgID",
+					"match_mapping_type": "*",
+					"mapping": {
+						"type": "long"
+					}
+				}
+			},
+			{
+				"numeric_fields": {
+					"path_match": "P:*.numField",
+					"match_mapping_type": "*",
+					"mapping": {
+						"type": "double",
+						"fields": {
+							"keyword": {
+								"type": "keyword",
+								"ignore_above": 256
+							}
+						}
+					}
+				}
+			},
+			{
+				"date_fields": {
+					"path_match": "P:*.datField",
+					"match_mapping_type": "*",
+					"mapping": {
+						"type": "double",
+						"fields": {
+							"keyword": {
+								"type": "keyword",
+								"ignore_above": 256
+							}
+						}
+					}
+				}
+			},
+			{
+				"date_fields_raw": {
+					"path_match": "P:*.dat_raw",
+					"match_mapping_type": "*",
+					"mapping": {
+						"type": "keyword"
+					}
+				}
+			},
+			{
+				"geo_fields": {
+					"path_match": "P:*.geoField",
+					"match_mapping_type": "string",
+					"mapping": {
+						"type": "keyword",
+						"fields": {
+							"point": {
+								"type": "geo_point"
+							}
+						}
+					}
+				}
+			},
+			{
+				"boolean_fields": {
+					"path_match": "P:*.booField",
+					"match_mapping_type": "boolean",
+					"mapping": {
+						"type": "boolean"
+					}
+				}
+			}
+		],
+		"properties": {
+			"noop": {
+				"type": "integer"
+			},
+			"text_copy": {
+				"type": "text"
+			},
+			"text_raw": {
+				"type": "text"
+			},
+			"subject.title": {
+				"type": "text",
+				"fields": {
+					"sort": {
+						"type": "keyword",
+						"normalizer": "standard_sort_normalizer",
+						"index": false
+					},
+					"keyword": {
+						"type": "keyword",
+						"ignore_above": 256
+					}
+				}
+			},
+			"subject.interwiki": {
+				"type": "text",
+				"fields": {
+					"keyword": {
+						"type": "keyword",
+						"ignore_above": 256
+					}
+				}
+			},
+			"subject.subobject": {
+				"type": "text",
+				"fields": {
+					"keyword": {
+						"type": "keyword",
+						"ignore_above": 256
+					}
+				}
+			},
+			"subject.sortkey": {
+				"type": "text",
+				"copy_to": "text_copy",
+				"fields": {
+					"sort": {
+						"type": "keyword",
+						"normalizer": "standard_sort_normalizer",
+						"index": false
+					},
+					"keyword": {
+						"type": "keyword",
+						"ignore_above": 256
+					},
+					"lowercase": {
+						"type": "keyword",
+						"normalizer": "lowercase_normalizer"
+					}
+				}
+			},
+			"subject.rev_id": {
+				"type": "integer"
+			}
+		}
+	}
+}

mediawiki/extensions/SemanticMediaWiki/data/elastic/smw-lookup.json

@@ -0,0 +1,15 @@
+{
+	"settings": {
+		"number_of_shards": 1,
+		"refresh_interval" : "1s",
+		"index.mapping.total_fields.limit": "3000",
+		"index.max_result_window": "50000"
+	},
+	"mappings": {
+		"properties": {
+			"id": {
+				"type": "long"
+			}
+		}
+	}
+}

mediawiki/extensions/SemanticMediaWiki/data/import/groups/predefined.properties.json

@@ -0,0 +1,76 @@
+{
+    "type": "PROPERTY_GROUP_SCHEMA",
+    "groups": {
+        "administrative_group": {
+            "canonical_name": "Administrative properties",
+            "message_key": "smw-property-group-label-administrative-properties",
+            "property_keys": [
+                "_MDAT",
+                "_CDAT",
+                "_NEWP",
+                "_LEDT",
+                "_DTITLE",
+                "_CHGPRO",
+                "_EDIP",
+                "_ERRC"
+            ]
+        },
+        "classification_group": {
+            "canonical_name": "Classification properties",
+            "message_key": "smw-property-group-label-classification-properties",
+            "property_keys": [
+                "_INST",
+                "_PPGR",
+                "_SUBP",
+                "_SUBC"
+            ]
+        },
+        "content_group": {
+            "canonical_name": "Content properties",
+            "message_key": "smw-property-group-label-content-properties",
+            "property_keys": [
+                "_SOBJ",
+                "_ASK",
+                "_MEDIA",
+                "_MIME",
+                "_ATTCH_LINK",
+                "_FILE_ATTCH",
+                "_CONT_TYPE",
+                "_CONT_AUTHOR",
+                "_CONT_LEN",
+                "_CONT_LANG",
+                "_CONT_TITLE",
+                "_CONT_DATE",
+                "_CONT_KEYW",
+                "_TRANS",
+                "_TRANS_SOURCE",
+                "_TRANS_GROUP"
+            ]
+        },
+        "declarative_group": {
+            "canonical_name": "Declarative properties",
+            "message_key" : "smw-property-group-label-declarative-properties",
+            "property_keys": [
+                "_TYPE",
+                "_UNIT",
+                "_IMPO",
+                "_CONV",
+                "_SERV",
+                "_PVAL",
+                "_LIST",
+                "_PREC",
+                "_PDESC",
+                "_PPLB",
+                "_PVAP",
+                "_PVALI",
+                "_PVUC",
+                "_PEID",
+                "_PEFU"
+            ]
+        }
+    },
+    "tags": [
+        "group",
+        "property group"
+    ]
+}

mediawiki/extensions/SemanticMediaWiki/data/import/groups/schema.json

@@ -0,0 +1,23 @@
+{
+    "type": "PROPERTY_GROUP_SCHEMA",
+    "groups": {
+        "schema_group": {
+            "canonical_name": "Schema properties",
+            "message_key": "smw-property-group-label-schema-group",
+            "property_keys": [
+                "_SCHEMA_TYPE",
+                "_SCHEMA_DEF",
+                "_SCHEMA_DESC",
+                "_SCHEMA_TAG",
+                "_SCHEMA_LINK",
+                "_FORMAT_SCHEMA",
+                "_CONSTRAINT_SCHEMA",
+                "_PROFILE_SCHEMA"
+            ]
+        }
+    },
+    "tags": [
+        "group",
+        "property group"
+    ]
+}

mediawiki/extensions/SemanticMediaWiki/data/import/properties/foaf.homepage.txt

@@ -0,0 +1,4 @@
+* [[Imported from::foaf:homepage]]
+* [[Property description::URL of the homepage of something, which is a general web resource.@en]] 
+
+[[Category:Imported vocabulary]] {{DISPLAYTITLE:foaf:homepage}}

mediawiki/extensions/SemanticMediaWiki/data/import/properties/foaf.knows.txt

@@ -0,0 +1,4 @@
+* [[Imported from::foaf:knows]]
+* [[Property description::A person known by this person (indicating some level of reciprocated interaction between the parties).@en]]
+
+[[Category:Imported vocabulary]] {{DISPLAYTITLE:foaf:knows}}

mediawiki/extensions/SemanticMediaWiki/data/import/properties/foaf.name.txt

@@ -0,0 +1,4 @@
+* [[Imported from::foaf:name]]
+* [[Property description::A name for some thing or agent.@en]]
+
+[[Category:Imported vocabulary]] {{DISPLAYTITLE:foaf:name}}