102 lines
3.9 KiB
PHP
102 lines
3.9 KiB
PHP
<?php
|
|
|
|
namespace Onoi\Tesa\Tests\Integration;
|
|
|
|
use Onoi\Tesa\SanitizerFactory;
|
|
use PHPUnit\Framework\TestCase;
|
|
|
|
/**
|
|
* @group onoi-tesa
|
|
*
|
|
* @license GPL-2.0-or-later
|
|
* @since 0.1
|
|
*
|
|
* @author mwjames
|
|
*/
|
|
class CombinedSanitizerTextStopwordTest extends TestCase {
|
|
|
|
/**
|
|
* @dataProvider textByLanguageProvider
|
|
*/
|
|
public function testByLanguage( $languageCode, $text, $expected ) {
|
|
$sanitizerFactory = new SanitizerFactory();
|
|
|
|
$sanitizer = $sanitizerFactory->newSanitizer( $text );
|
|
$sanitizer->toLowercase();
|
|
|
|
$text = $sanitizer->sanitizeWith(
|
|
$sanitizerFactory->newGenericRegExTokenizer(),
|
|
$sanitizerFactory->newCdbStopwordAnalyzer( $languageCode ),
|
|
$sanitizerFactory->newNullSynonymizer()
|
|
);
|
|
|
|
$this->assertEquals(
|
|
$expected,
|
|
$text
|
|
);
|
|
}
|
|
|
|
public function textByLanguageProvider() {
|
|
// https://en.wikipedia.org/wiki/Stop_words
|
|
$provider[] = [
|
|
'en',
|
|
//
|
|
'In computing, stop words are words which are filtered out before or after processing of ' .
|
|
'natural language data (text).[1] Though stop words usually refer to the most common words ' .
|
|
'in a language, there is no single universal list of stop words used by all natural language ' .
|
|
'processing tools, and indeed not all tools even use such a list. Some tools specifically avoid ' .
|
|
'removing these stop words to support phrase search.',
|
|
//
|
|
'computing stop words filtered processing natural language data text stop words refer common ' .
|
|
'words language single universal list stop words natural language processing tools list tools ' .
|
|
'specifically avoid removing stop words support phrase search'
|
|
];
|
|
|
|
// https://es.wikipedia.org/wiki/Palabra_vac%C3%ADa
|
|
$provider[] = [
|
|
'es',
|
|
//
|
|
'Palabras vacías es el nombre que reciben las palabras sin significado como artículos, pronombres, ' .
|
|
'preposiciones, etc. que son filtradas antes o después del procesamiento de datos en lenguaje natural ' .
|
|
'(texto). A Hans Peter Luhn, uno de los pioneros en recuperación de información, se le atribuye la ' .
|
|
'acuñación de la locución inglesa stop words y el uso del concepto en su diseño. Está controlada por ' .
|
|
'introducción humana y no automática.',
|
|
//
|
|
'palabras vacías nombre que reciben palabras significado artículos pronombres preposiciones etc que ' .
|
|
'son filtradas después del procesamiento datos lenguaje natural texto hans peter luhn pioneros ' .
|
|
'recuperación información atribuye acuñación locución inglesa stop words del concepto diseño está ' .
|
|
'controlada introducción humana automática'
|
|
];
|
|
|
|
// https://de.wikipedia.org/wiki/Stoppwort
|
|
$provider[] = [
|
|
'de',
|
|
//
|
|
'Stoppwörter nennt man im Information Retrieval Wörter, die bei einer Volltextindexierung nicht beachtet ' .
|
|
'werden, da sie sehr häufig auftreten und gewöhnlich keine Relevanz für die Erfassung des Dokumentinhalts ' .
|
|
'besitzen.',
|
|
//
|
|
'stoppwörter nennt information retrieval wörter volltextindexierung beachtet häufig auftreten gewöhnlich ' .
|
|
'relevanz erfassung dokumentinhalts besitzen'
|
|
];
|
|
|
|
// https://en.wikipedia.org/wiki/Query_expansion
|
|
$provider[] = [
|
|
'en',
|
|
//
|
|
"The goal of query expansion in this regard is by increasing recall, precision can potentially increase " .
|
|
"(rather than decrease as mathematically equated), by including in the result set pages which are more " .
|
|
"relevant (of higher quality), or at least equally relevant. Pages which would not be included in the " .
|
|
"result set, which have the potential to be more relevant to the user's desired query, are included, and " .
|
|
"without query expansion would not have, regardless of relevance.",
|
|
//
|
|
"goal query expansion regard increasing recall precision potentially increase decrease mathematically " .
|
|
"equated including result set pages relevant quality equally relevant pages included result set potential " .
|
|
"relevant user desired query included query expansion relevance"
|
|
];
|
|
|
|
return $provider;
|
|
}
|
|
|
|
}
|