Коммит 5ab5623b создал по автору OMP Education's avatar OMP Education Зафиксировано автором Vladislav Larionov
Просмотр файлов

[Project] Update the libreoffice source repository and adapt the project for Aurora 5

владелец 9c9825fa
Скрыть пробелы
Построчно Рядом
libs/hunspell-1.7.0/src/hunspell/hunvisapi.h 0 → 100644
Просмотр файла @ 5ab5623b
#ifndef HUNSPELL_VISIBILITY_H_
#define HUNSPELL_VISIBILITY_H_
#if defined(HUNSPELL_STATIC)
# define LIBHUNSPELL_DLL_EXPORTED
#elif defined(_WIN32)
# if defined(BUILDING_LIBHUNSPELL)
# define LIBHUNSPELL_DLL_EXPORTED __declspec(dllexport)
# else
# define LIBHUNSPELL_DLL_EXPORTED __declspec(dllimport)
# endif
#elif defined(BUILDING_LIBHUNSPELL) && 1
# define LIBHUNSPELL_DLL_EXPORTED __attribute__((__visibility__("default")))
#else
# define LIBHUNSPELL_DLL_EXPORTED
#endif
#endif
libs/hunspell-1.7.0/src/hunspell/hunvisapi.h.in 0 → 100644
Просмотр файла @ 5ab5623b
#ifndef HUNSPELL_VISIBILITY_H_
#define HUNSPELL_VISIBILITY_H_
#if defined(HUNSPELL_STATIC)
# define LIBHUNSPELL_DLL_EXPORTED
#elif defined(_WIN32)
# if defined(BUILDING_LIBHUNSPELL)
# define LIBHUNSPELL_DLL_EXPORTED __declspec(dllexport)
# else
# define LIBHUNSPELL_DLL_EXPORTED __declspec(dllimport)
# endif
#elif defined(BUILDING_LIBHUNSPELL) && @HAVE_VISIBILITY@
# define LIBHUNSPELL_DLL_EXPORTED __attribute__((__visibility__("default")))
#else
# define LIBHUNSPELL_DLL_EXPORTED
#endif
#endif
libs/hunspell-1.7.0/src/hunspell/hunzip.cxx 0 → 100644
Просмотр файла @ 5ab5623b
/* ***** BEGIN LICENSE BLOCK *****
* Version: MPL 1.1/GPL 2.0/LGPL 2.1
*
* Copyright (C) 2002-2017 Németh László
*
* The contents of this file are subject to the Mozilla Public License Version
* 1.1 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* http://www.mozilla.org/MPL/
*
* Software distributed under the License is distributed on an "AS IS" basis,
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
* for the specific language governing rights and limitations under the
* License.
*
* Hunspell is based on MySpell which is Copyright (C) 2002 Kevin Hendricks.
*
* Contributor(s): David Einstein, Davide Prina, Giuseppe Modugno,
* Gianluca Turconi, Simon Brouwer, Noll János, Bíró Árpád,
* Goldman Eleonóra, Sarlós Tamás, Bencsáth Boldizsár, Halácsy Péter,
* Dvornik László, Gefferth András, Nagy Viktor, Varga Dániel, Chris Halls,
* Rene Engelhard, Bram Moolenaar, Dafydd Jones, Harri Pitkänen
*
* Alternatively, the contents of this file may be used under the terms of
* either the GNU General Public License Version 2 or later (the "GPL"), or
* the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
* in which case the provisions of the GPL or the LGPL are applicable instead
* of those above. If you wish to allow use of your version of this file only
* under the terms of either the GPL or the LGPL, and not to allow others to
* use your version of this file under the terms of the MPL, indicate your
* decision by deleting the provisions above and replace them with the notice
* and other provisions required by the GPL or the LGPL. If you do not delete
* the provisions above, a recipient may use your version of this file under
* the terms of any one of the MPL, the GPL or the LGPL.
*
* ***** END LICENSE BLOCK ***** */
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include "hunzip.hxx"
#include "csutil.hxx"
#define CODELEN 65536
#define BASEBITREC 5000
#define UNCOMPRESSED '\002'
#define MAGIC "hz0"
#define MAGIC_ENCRYPT "hz1"
#define MAGICLEN (sizeof(MAGIC) - 1)
int Hunzip::fail(const char* err, const char* par) {
fprintf(stderr, err, par);
return -1;
}
Hunzip::Hunzip(const char* file, const char* key)
: bufsiz(0), lastbit(0), inc(0), inbits(0), outc(0) {
in[0] = out[0] = line[0] = '\0';
filename = mystrdup(file);
if (getcode(key) == -1)
bufsiz = -1;
else
bufsiz = getbuf();
}
int Hunzip::getcode(const char* key) {
unsigned char c[2];
int i, j, n;
int allocatedbit = BASEBITREC;
const char* enc = key;
if (!filename)
return -1;
myopen(fin, filename, std::ios_base::in | std::ios_base::binary);
if (!fin.is_open())
return -1;
// read magic number
if (!fin.read(in, 3) ||
!(strncmp(MAGIC, in, MAGICLEN) == 0 ||
strncmp(MAGIC_ENCRYPT, in, MAGICLEN) == 0)) {
return fail(MSG_FORMAT, filename);
}
// check encryption
if (strncmp(MAGIC_ENCRYPT, in, MAGICLEN) == 0) {
unsigned char cs;
if (!key)
return fail(MSG_KEY, filename);
if (!fin.read(reinterpret_cast<char*>(c), 1))
return fail(MSG_FORMAT, filename);
for (cs = 0; *enc; enc++)
cs ^= *enc;
if (cs != c[0])
return fail(MSG_KEY, filename);
enc = key;
} else
key = NULL;
// read record count
if (!fin.read(reinterpret_cast<char*>(c), 2))
return fail(MSG_FORMAT, filename);
if (key) {
c[0] ^= *enc;
if (*(++enc) == '\0')
enc = key;
c[1] ^= *enc;
}
n = ((int)c[0] << 8) + c[1];
dec.resize(BASEBITREC);
dec[0].v[0] = 0;
dec[0].v[1] = 0;
// read codes
for (i = 0; i < n; i++) {
unsigned char l;
if (!fin.read(reinterpret_cast<char*>(c), 2))
return fail(MSG_FORMAT, filename);
if (key) {
if (*(++enc) == '\0')
enc = key;
c[0] ^= *enc;
if (*(++enc) == '\0')
enc = key;
c[1] ^= *enc;
}
if (!fin.read(reinterpret_cast<char*>(&l), 1))
return fail(MSG_FORMAT, filename);
if (key) {
if (*(++enc) == '\0')
enc = key;
l ^= *enc;
}
if (!fin.read(in, l / 8 + 1))
return fail(MSG_FORMAT, filename);
if (key)
for (j = 0; j <= l / 8; j++) {
if (*(++enc) == '\0')
enc = key;
in[j] ^= *enc;
}
int p = 0;
for (j = 0; j < l; j++) {
int b = (in[j / 8] & (1 << (7 - (j % 8)))) ? 1 : 0;
int oldp = p;
p = dec[p].v[b];
if (p == 0) {
lastbit++;
if (lastbit == allocatedbit) {
allocatedbit += BASEBITREC;
dec.resize(allocatedbit);
}
dec[lastbit].v[0] = 0;
dec[lastbit].v[1] = 0;
dec[oldp].v[b] = lastbit;
p = lastbit;
}
}
dec[p].c[0] = c[0];
dec[p].c[1] = c[1];
}
return 0;
}
Hunzip::~Hunzip() {
if (filename)
free(filename);
}
int Hunzip::getbuf() {
int p = 0;
int o = 0;
do {
if (inc == 0) {
fin.read(in, BUFSIZE);
inbits = int(fin.gcount() * 8);
}
for (; inc < inbits; inc++) {
int b = (in[inc / 8] & (1 << (7 - (inc % 8)))) ? 1 : 0;
int oldp = p;
p = dec[p].v[b];
if (p == 0) {
if (oldp == lastbit) {
fin.close();
// add last odd byte
if (dec[lastbit].c[0])
out[o++] = dec[lastbit].c[1];
return o;
}
out[o++] = dec[oldp].c[0];
out[o++] = dec[oldp].c[1];
if (o == BUFSIZE)
return o;
p = dec[p].v[b];
}
}
inc = 0;
} while (inbits == BUFSIZE * 8);
return fail(MSG_FORMAT, filename);
}
bool Hunzip::getline(std::string& dest) {
char linebuf[BUFSIZE];
int l = 0, eol = 0, left = 0, right = 0;
if (bufsiz == -1)
return false;
while (l < bufsiz && !eol) {
linebuf[l++] = out[outc];
switch (out[outc]) {
case '\t':
break;
case 31: { // escape
if (++outc == bufsiz) {
bufsiz = getbuf();
outc = 0;
}
linebuf[l - 1] = out[outc];
break;
}
case ' ':
break;
default:
if (((unsigned char)out[outc]) < 47) {
if (out[outc] > 32) {
right = out[outc] - 31;
if (++outc == bufsiz) {
bufsiz = getbuf();
outc = 0;
}
}
if (out[outc] == 30)
left = 9;
else
left = out[outc];
linebuf[l - 1] = '\n';
eol = 1;
}
}
if (++outc == bufsiz) {
outc = 0;
bufsiz = fin.is_open() ? getbuf() : -1;
}
}
if (right)
strcpy(linebuf + l - 1, line + strlen(line) - right - 1);
else
linebuf[l] = '\0';
strcpy(line + left, linebuf);
dest.assign(line);
return true;
}
libs/hunspell-1.7.0/src/hunspell/hunzip.hxx 0 → 100644
Просмотр файла @ 5ab5623b
/* ***** BEGIN LICENSE BLOCK *****
* Version: MPL 1.1/GPL 2.0/LGPL 2.1
*
* Copyright (C) 2002-2017 Németh László
*
* The contents of this file are subject to the Mozilla Public License Version
* 1.1 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* http://www.mozilla.org/MPL/
*
* Software distributed under the License is distributed on an "AS IS" basis,
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
* for the specific language governing rights and limitations under the
* License.
*
* Hunspell is based on MySpell which is Copyright (C) 2002 Kevin Hendricks.
*
* Contributor(s): David Einstein, Davide Prina, Giuseppe Modugno,
* Gianluca Turconi, Simon Brouwer, Noll János, Bíró Árpád,
* Goldman Eleonóra, Sarlós Tamás, Bencsáth Boldizsár, Halácsy Péter,
* Dvornik László, Gefferth András, Nagy Viktor, Varga Dániel, Chris Halls,
* Rene Engelhard, Bram Moolenaar, Dafydd Jones, Harri Pitkänen
*
* Alternatively, the contents of this file may be used under the terms of
* either the GNU General Public License Version 2 or later (the "GPL"), or
* the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
* in which case the provisions of the GPL or the LGPL are applicable instead
* of those above. If you wish to allow use of your version of this file only
* under the terms of either the GPL or the LGPL, and not to allow others to
* use your version of this file under the terms of the MPL, indicate your
* decision by deleting the provisions above and replace them with the notice
* and other provisions required by the GPL or the LGPL. If you do not delete
* the provisions above, a recipient may use your version of this file under
* the terms of any one of the MPL, the GPL or the LGPL.
*
* ***** END LICENSE BLOCK ***** */
/* hunzip: file decompression for sorted dictionaries with optional encryption,
* algorithm: prefix-suffix encoding and 16-bit Huffman encoding */
#ifndef HUNZIP_HXX_
#define HUNZIP_HXX_
#include "hunvisapi.h"
#include <stdio.h>
#include <fstream>
#include <vector>
#define BUFSIZE 65536
#define HZIP_EXTENSION ".hz"
#define MSG_OPEN "error: %s: cannot open\n"
#define MSG_FORMAT "error: %s: not in hzip format\n"
#define MSG_MEMORY "error: %s: missing memory\n"
#define MSG_KEY "error: %s: missing or bad password\n"
struct bit {
unsigned char c[2];
int v[2];
};
class LIBHUNSPELL_DLL_EXPORTED Hunzip {
private:
Hunzip(const Hunzip&);
Hunzip& operator=(const Hunzip&);
protected:
char* filename;
std::ifstream fin;
int bufsiz, lastbit, inc, inbits, outc;
std::vector<bit> dec; // code table
char in[BUFSIZE]; // input buffer
char out[BUFSIZE + 1]; // Huffman-decoded buffer
char line[BUFSIZE + 50]; // decoded line
int getcode(const char* key);
int getbuf();
int fail(const char* err, const char* par);
public:
Hunzip(const char* filename, const char* key = NULL);
~Hunzip();
bool is_open() { return fin.is_open(); }
bool getline(std::string& dest);
};
#endif
libs/hunspell-1.7.0/src/hunspell/langnum.hxx 0 → 100644
Просмотр файла @ 5ab5623b
/* ***** BEGIN LICENSE BLOCK *****
* Version: MPL 1.1/GPL 2.0/LGPL 2.1
*
* Copyright (C) 2002-2017 Németh László
*
* The contents of this file are subject to the Mozilla Public License Version
* 1.1 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* http://www.mozilla.org/MPL/
*
* Software distributed under the License is distributed on an "AS IS" basis,
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
* for the specific language governing rights and limitations under the
* License.
*
* Hunspell is based on MySpell which is Copyright (C) 2002 Kevin Hendricks.
*
* Contributor(s): David Einstein, Davide Prina, Giuseppe Modugno,
* Gianluca Turconi, Simon Brouwer, Noll János, Bíró Árpád,
* Goldman Eleonóra, Sarlós Tamás, Bencsáth Boldizsár, Halácsy Péter,
* Dvornik László, Gefferth András, Nagy Viktor, Varga Dániel, Chris Halls,
* Rene Engelhard, Bram Moolenaar, Dafydd Jones, Harri Pitkänen
*
* Alternatively, the contents of this file may be used under the terms of
* either the GNU General Public License Version 2 or later (the "GPL"), or
* the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
* in which case the provisions of the GPL or the LGPL are applicable instead
* of those above. If you wish to allow use of your version of this file only
* under the terms of either the GPL or the LGPL, and not to allow others to
* use your version of this file under the terms of the MPL, indicate your
* decision by deleting the provisions above and replace them with the notice
* and other provisions required by the GPL or the LGPL. If you do not delete
* the provisions above, a recipient may use your version of this file under
* the terms of any one of the MPL, the GPL or the LGPL.
*
* ***** END LICENSE BLOCK ***** */
#ifndef LANGNUM_HXX_
#define LANGNUM_HXX_
/*
language numbers for language specific codes
see https://wiki.openoffice.org/w/index.php?title=Languages&oldid=230199
*/
enum {
LANG_ar = 96,
LANG_az = 100, // custom number
LANG_bg = 41,
LANG_ca = 37,
LANG_crh = 102, // custom number
LANG_cs = 42,
LANG_da = 45,
LANG_de = 49,
LANG_el = 30,
LANG_en = 01,
LANG_es = 34,
LANG_eu = 10,
LANG_fr = 02,
LANG_gl = 38,
LANG_hr = 78,
LANG_hu = 36,
LANG_it = 39,
LANG_la = 99, // custom number
LANG_lv = 101, // custom number
LANG_nl = 31,
LANG_pl = 48,
LANG_pt = 03,
LANG_ru = 07,
LANG_sv = 50,
LANG_tr = 90,
LANG_uk = 80,
LANG_xx = 999
};
#endif
libs/hunspell-1.7.0/src/hunspell/phonet.cxx 0 → 100644
Просмотр файла @ 5ab5623b
/* phonetic.c - generic replacement aglogithms for phonetic transformation
Copyright (C) 2000 Bjoern Jacke
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License version 2.1 as published by the Free Software Foundation;
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; If not, see
<http://www.gnu.org/licenses/>.
Changelog:
2000-01-05 Bjoern Jacke <bjoern at j3e.de>
Initial Release insprired by the article about phonetic
transformations out of c't 25/1999
2007-07-26 Bjoern Jacke <bjoern at j3e.de>
Released under MPL/GPL/LGPL tri-license for Hunspell
2007-08-23 Laszlo Nemeth <nemeth at OOo>
Porting from Aspell to Hunspell using C-like structs
*/
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <ctype.h>
#include "csutil.hxx"
#include "phonet.hxx"
void init_phonet_hash(phonetable& parms) {
for (int i = 0; i < HASHSIZE; i++) {
parms.hash[i] = -1;
}
for (int i = 0; parms.rules[i][0] != '\0'; i += 2) {
/** set hash value **/
int k = (unsigned char)parms.rules[i][0];
if (parms.hash[k] < 0) {
parms.hash[k] = i;
}
}
}
// like strcpy but safe if the strings overlap
// but only if dest < src
static inline void strmove(char* dest, char* src) {
while (*src)
*dest++ = *src++;
*dest = '\0';
}
static int myisalpha(char ch) {
if ((unsigned char)ch < 128)
return isalpha(ch);
return 1;
}
/* Do phonetic transformation. */
/* phonetic transcription algorithm */
/* see: http://aspell.net/man-html/Phonetic-Code.html */
/* convert string to uppercase before this call */
std::string phonet(const std::string& inword, phonetable& parms) {
int i, k = 0, p, z;
int k0, n0, p0 = -333;
char c;
typedef unsigned char uchar;
size_t len = inword.size();
if (len > MAXPHONETUTF8LEN)
return std::string();
char word[MAXPHONETUTF8LEN + 1];
strncpy(word, inword.c_str(), MAXPHONETUTF8LEN);
word[MAXPHONETUTF8LEN] = '\0';
std::string target;
/** check word **/
i = z = 0;
while ((c = word[i]) != '\0') {
int n = parms.hash[(uchar)c];
int z0 = 0;
if (n >= 0 && !parms.rules[n].empty()) {
/** check all rules for the same letter **/
while (parms.rules[n][0] == c) {
/** check whole string **/
k = 1; /** number of found letters **/
p = 5; /** default priority **/
const char*s = parms.rules[n].c_str();
s++; /** important for (see below) "*(s-1)" **/
while (*s != '\0' && word[i + k] == *s && !isdigit((unsigned char)*s) &&
strchr("(-<^$", *s) == NULL) {
k++;
s++;
}
if (*s == '(') {
/** check letters in "(..)" **/
if (myisalpha(word[i + k]) // ...could be implied?
&& strchr(s + 1, word[i + k]) != NULL) {
k++;
while (*s != ')')
s++;
s++;
}
}
p0 = (int)*s;
k0 = k;
while (*s == '-' && k > 1) {
k--;
s++;
}
if (*s == '<')
s++;
if (isdigit((unsigned char)*s)) {
/** determine priority **/
p = *s - '0';
s++;
}
if (*s == '^' && *(s + 1) == '^')
s++;
if (*s == '\0' || (*s == '^' && (i == 0 || !myisalpha(word[i - 1])) &&
(*(s + 1) != '$' || (!myisalpha(word[i + k0])))) ||
(*s == '$' && i > 0 && myisalpha(word[i - 1]) &&
(!myisalpha(word[i + k0])))) {
/** search for followup rules, if: **/
/** parms.followup and k > 1 and NO '-' in searchstring **/
char c0 = word[i + k - 1];
n0 = parms.hash[(uchar)c0];
// if (parms.followup && k > 1 && n0 >= 0
if (k > 1 && n0 >= 0 && p0 != (int)'-' && word[i + k] != '\0' && !parms.rules[n0].empty()) {
/** test follow-up rule for "word[i+k]" **/
while (parms.rules[n0][0] == c0) {
/** check whole string **/
k0 = k;
p0 = 5;
s = parms.rules[n0].c_str();
s++;
while (*s != '\0' && word[i + k0] == *s &&
!isdigit((unsigned char)*s) &&
strchr("(-<^$", *s) == NULL) {
k0++;
s++;
}
if (*s == '(') {
/** check letters **/
if (myisalpha(word[i + k0]) &&
strchr(s + 1, word[i + k0]) != NULL) {
k0++;
while (*s != ')' && *s != '\0')
s++;
if (*s == ')')
s++;
}
}
while (*s == '-') {
/** "k0" gets NOT reduced **/
/** because "if (k0 == k)" **/
s++;
}
if (*s == '<')
s++;
if (isdigit((unsigned char)*s)) {
p0 = *s - '0';
s++;
}
if (*s == '\0'
/** *s == '^' cuts **/
|| (*s == '$' && !myisalpha(word[i + k0]))) {
if (k0 == k) {
/** this is just a piece of the string **/
n0 += 2;
continue;
}
if (p0 < p) {
/** priority too low **/
n0 += 2;
continue;
}
/** rule fits; stop search **/
break;
}
n0 += 2;
} /** End of "while (parms.rules[n0][0] == c0)" **/
if (p0 >= p && parms.rules[n0][0] == c0) {
n += 2;
continue;
}
} /** end of follow-up stuff **/
/** replace string **/
s = parms.rules[n + 1].c_str();
p0 = (!parms.rules[n].empty() &&
strchr(parms.rules[n].c_str() + 1, '<') != NULL)
? 1
: 0;
if (p0 == 1 && z == 0) {
/** rule with '<' is used **/
if (!target.empty() && *s != '\0' &&
(target[target.size()-1] == c || target[target.size()-1] == *s)) {
target.erase(target.size() - 1);
}
z0 = 1;
z = 1;
k0 = 0;
while (*s != '\0' && word[i + k0] != '\0') {
word[i + k0] = *s;
k0++;
s++;
}
if (k > k0)
strmove(&word[0] + i + k0, &word[0] + i + k);
/** new "actual letter" **/
c = word[i];
} else { /** no '<' rule used **/
i += k - 1;
z = 0;
while (*s != '\0' && *(s + 1) != '\0' && target.size() < len) {
if (target.empty() || target[target.size()-1] != *s) {
target.push_back(*s);
}
s++;
}
/** new "actual letter" **/
c = *s;
if (!parms.rules[n].empty() &&
strstr(parms.rules[n].c_str() + 1, "^^") != NULL) {
if (c != '\0') {
target.push_back(c);
}
strmove(&word[0], &word[0] + i + 1);
i = 0;
z0 = 1;
}
}
break;
} /** end of follow-up stuff **/
n += 2;
} /** end of while (parms.rules[n][0] == c) **/
} /** end of if (n >= 0) **/
if (z0 == 0) {
if (k && !p0 && target.size() < len && c != '\0') {
/** condense only double letters **/
target.push_back(c);
/// printf("\n setting \n");
}
i++;
z = 0;
k = 0;
}
} /** end of while ((c = word[i]) != '\0') **/
return target;
} /** end of function "phonet" **/
libs/hunspell-1.7.0/src/hunspell/phonet.hxx 0 → 100644
Просмотр файла @ 5ab5623b
/* phonetic.c - generic replacement aglogithms for phonetic transformation
Copyright (C) 2000 Bjoern Jacke
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License version 2.1 as published by the Free Software Foundation;
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; If not, see
<http://www.gnu.org/licenses/>.
Changelog:
2000-01-05 Bjoern Jacke <bjoern at j3e.de>
Initial Release insprired by the article about phonetic
transformations out of c't 25/1999
2007-07-26 Bjoern Jacke <bjoern at j3e.de>
Released under MPL/GPL/LGPL tri-license for Hunspell
2007-08-23 Laszlo Nemeth <nemeth at OOo>
Porting from Aspell to Hunspell using C-like structs
*/
#ifndef PHONET_HXX_
#define PHONET_HXX_
#define HASHSIZE 256
#define MAXPHONETLEN 256
#define MAXPHONETUTF8LEN (MAXPHONETLEN * 4)
#include "hunvisapi.h"
struct phonetable {
char utf8;
std::vector<std::string> rules;
int hash[HASHSIZE];
};
LIBHUNSPELL_DLL_EXPORTED void init_phonet_hash(phonetable& parms);
LIBHUNSPELL_DLL_EXPORTED std::string phonet(const std::string& inword,
phonetable& phone);
#endif
libs/hunspell-1.7.0/src/hunspell/replist.cxx 0 → 100644
Просмотр файла @ 5ab5623b
/* ***** BEGIN LICENSE BLOCK *****
* Version: MPL 1.1/GPL 2.0/LGPL 2.1
*
* Copyright (C) 2002-2017 Németh László
*
* The contents of this file are subject to the Mozilla Public License Version
* 1.1 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* http://www.mozilla.org/MPL/
*
* Software distributed under the License is distributed on an "AS IS" basis,
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
* for the specific language governing rights and limitations under the
* License.
*
* Hunspell is based on MySpell which is Copyright (C) 2002 Kevin Hendricks.
*
* Contributor(s): David Einstein, Davide Prina, Giuseppe Modugno,
* Gianluca Turconi, Simon Brouwer, Noll János, Bíró Árpád,
* Goldman Eleonóra, Sarlós Tamás, Bencsáth Boldizsár, Halácsy Péter,
* Dvornik László, Gefferth András, Nagy Viktor, Varga Dániel, Chris Halls,
* Rene Engelhard, Bram Moolenaar, Dafydd Jones, Harri Pitkänen
*
* Alternatively, the contents of this file may be used under the terms of
* either the GNU General Public License Version 2 or later (the "GPL"), or
* the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
* in which case the provisions of the GPL or the LGPL are applicable instead
* of those above. If you wish to allow use of your version of this file only
* under the terms of either the GPL or the LGPL, and not to allow others to
* use your version of this file under the terms of the MPL, indicate your
* decision by deleting the provisions above and replace them with the notice
* and other provisions required by the GPL or the LGPL. If you do not delete
* the provisions above, a recipient may use your version of this file under
* the terms of any one of the MPL, the GPL or the LGPL.
*
* ***** END LICENSE BLOCK ***** */
/*
* Copyright 2002 Kevin B. Hendricks, Stratford, Ontario, Canada
* And Contributors. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* 3. All modifications to the source code must be clearly marked as
* such. Binary redistributions based on modified source code
* must be clearly marked as modified versions in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY KEVIN B. HENDRICKS AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
* KEVIN B. HENDRICKS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <limits>
#include "replist.hxx"
#include "csutil.hxx"
RepList::RepList(int n) {
dat = (replentry**)malloc(sizeof(replentry*) * n);
if (dat == 0)
size = 0;
else
size = n;
pos = 0;
}
RepList::~RepList() {
for (int i = 0; i < pos; i++) {
delete dat[i];
}
free(dat);
}
replentry* RepList::item(int n) {
return dat[n];
}
int RepList::find(const char* word) {
int p1 = 0;
int p2 = pos - 1;
int ret = -1;
while (p1 <= p2) {
int m = ((unsigned)p1 + (unsigned)p2) >> 1;
int c = strncmp(word, dat[m]->pattern.c_str(), dat[m]->pattern.size());
if (c < 0)
p2 = m - 1;
else if (c > 0)
p1 = m + 1;
else { // scan in the right half for a longer match
ret = m;
p1 = m + 1;
}
}
return ret;
}
std::string RepList::replace(const char* word, int ind, bool atstart) {
int type = atstart ? 1 : 0;
if (ind < 0)
return std::string();
if (strlen(word) == dat[ind]->pattern.size())
type = atstart ? 3 : 2;
while (type && dat[ind]->outstrings[type].empty())
type = (type == 2 && !atstart) ? 0 : type - 1;
return dat[ind]->outstrings[type];
}
int RepList::add(const std::string& in_pat1, const std::string& pat2) {
if (pos >= size || in_pat1.empty() || pat2.empty()) {
return 1;
}
// analyse word context
int type = 0;
std::string pat1(in_pat1);
if (pat1[0] == '_') {
pat1.erase(0, 1);
type = 1;
}
if (!pat1.empty() && pat1[pat1.size() - 1] == '_') {
type = type + 2;
pat1.erase(pat1.size() - 1);
}
mystrrep(pat1, "_", " ");
// find existing entry
int m = find(pat1.c_str());
if (m >= 0 && dat[m]->pattern == pat1) {
// since already used
dat[m]->outstrings[type] = pat2;
mystrrep(dat[m]->outstrings[type], "_", " ");
return 0;
}
// make a new entry if none exists
replentry* r = new replentry;
if (r == NULL)
return 1;
r->pattern = pat1;
r->outstrings[type] = pat2;
mystrrep(r->outstrings[type], "_", " ");
dat[pos++] = r;
// sort to the right place in the list
int i;
for (i = pos - 1; i > 0; i--) {
if (strcmp(r->pattern.c_str(), dat[i - 1]->pattern.c_str()) < 0) {
dat[i] = dat[i - 1];
} else
break;
}
dat[i] = r;
return 0;
}
bool RepList::conv(const std::string& in_word, std::string& dest) {
dest.clear();
size_t wordlen = in_word.size();
const char* word = in_word.c_str();
bool change = false;
for (size_t i = 0; i < wordlen; ++i) {
int n = find(word + i);
std::string l = replace(word + i, n, i == 0);
if (!l.empty()) {
dest.append(l);
i += dat[n]->pattern.size() - 1;
change = true;
} else {
dest.push_back(word[i]);
}
}
return change;
}
libs/hunspell-1.7.0/src/hunspell/replist.hxx 0 → 100644
Просмотр файла @ 5ab5623b
/* ***** BEGIN LICENSE BLOCK *****
* Version: MPL 1.1/GPL 2.0/LGPL 2.1
*
* Copyright (C) 2002-2017 Németh László
*
* The contents of this file are subject to the Mozilla Public License Version
* 1.1 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* http://www.mozilla.org/MPL/
*
* Software distributed under the License is distributed on an "AS IS" basis,
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
* for the specific language governing rights and limitations under the
* License.
*
* Hunspell is based on MySpell which is Copyright (C) 2002 Kevin Hendricks.
*
* Contributor(s): David Einstein, Davide Prina, Giuseppe Modugno,
* Gianluca Turconi, Simon Brouwer, Noll János, Bíró Árpád,
* Goldman Eleonóra, Sarlós Tamás, Bencsáth Boldizsár, Halácsy Péter,
* Dvornik László, Gefferth András, Nagy Viktor, Varga Dániel, Chris Halls,
* Rene Engelhard, Bram Moolenaar, Dafydd Jones, Harri Pitkänen
*
* Alternatively, the contents of this file may be used under the terms of
* either the GNU General Public License Version 2 or later (the "GPL"), or
* the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
* in which case the provisions of the GPL or the LGPL are applicable instead
* of those above. If you wish to allow use of your version of this file only
* under the terms of either the GPL or the LGPL, and not to allow others to
* use your version of this file under the terms of the MPL, indicate your
* decision by deleting the provisions above and replace them with the notice
* and other provisions required by the GPL or the LGPL. If you do not delete
* the provisions above, a recipient may use your version of this file under
* the terms of any one of the MPL, the GPL or the LGPL.
*
* ***** END LICENSE BLOCK ***** */
/*
* Copyright 2002 Kevin B. Hendricks, Stratford, Ontario, Canada
* And Contributors. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* 3. All modifications to the source code must be clearly marked as
* such. Binary redistributions based on modified source code
* must be clearly marked as modified versions in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY KEVIN B. HENDRICKS AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
* KEVIN B. HENDRICKS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
/* string replacement list class */
#ifndef REPLIST_HXX_
#define REPLIST_HXX_
#include "w_char.hxx"
#include <string>
#include <vector>
class RepList {
private:
RepList(const RepList&);
RepList& operator=(const RepList&);
protected:
replentry** dat;
int size;
int pos;
public:
explicit RepList(int n);
~RepList();
int add(const std::string& pat1, const std::string& pat2);
replentry* item(int n);
int find(const char* word);
std::string replace(const char* word, int n, bool atstart);
bool conv(const std::string& word, std::string& dest);
};
#endif
libs/hunspell-1.7.0/src/hunspell/suggestmgr.cxx 0 → 100644
Просмотр файла @ 5ab5623b
/* ***** BEGIN LICENSE BLOCK *****
* Version: MPL 1.1/GPL 2.0/LGPL 2.1
*
* Copyright (C) 2002-2017 Németh László
*
* The contents of this file are subject to the Mozilla Public License Version
* 1.1 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* http://www.mozilla.org/MPL/
*
* Software distributed under the License is distributed on an "AS IS" basis,
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
* for the specific language governing rights and limitations under the
* License.
*
* Hunspell is based on MySpell which is Copyright (C) 2002 Kevin Hendricks.
*
* Contributor(s): David Einstein, Davide Prina, Giuseppe Modugno,
* Gianluca Turconi, Simon Brouwer, Noll János, Bíró Árpád,
* Goldman Eleonóra, Sarlós Tamás, Bencsáth Boldizsár, Halácsy Péter,
* Dvornik László, Gefferth András, Nagy Viktor, Varga Dániel, Chris Halls,
* Rene Engelhard, Bram Moolenaar, Dafydd Jones, Harri Pitkänen
*
* Alternatively, the contents of this file may be used under the terms of
* either the GNU General Public License Version 2 or later (the "GPL"), or
* the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
* in which case the provisions of the GPL or the LGPL are applicable instead
* of those above. If you wish to allow use of your version of this file only
* under the terms of either the GPL or the LGPL, and not to allow others to
* use your version of this file under the terms of the MPL, indicate your
* decision by deleting the provisions above and replace them with the notice
* and other provisions required by the GPL or the LGPL. If you do not delete
* the provisions above, a recipient may use your version of this file under
* the terms of any one of the MPL, the GPL or the LGPL.
*
* ***** END LICENSE BLOCK ***** */
/*
* Copyright 2002 Kevin B. Hendricks, Stratford, Ontario, Canada
* And Contributors. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* 3. All modifications to the source code must be clearly marked as
* such. Binary redistributions based on modified source code
* must be clearly marked as modified versions in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY KEVIN B. HENDRICKS AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
* KEVIN B. HENDRICKS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <ctype.h>
#include <time.h>
#include "suggestmgr.hxx"
#include "htypes.hxx"
#include "csutil.hxx"
const w_char W_VLINE = {'\0', '|'};
#define MAX_CHAR_DISTANCE 4
SuggestMgr::SuggestMgr(const char* tryme, unsigned int maxn, AffixMgr* aptr) {
// register affix manager and check in string of chars to
// try when building candidate suggestions
pAMgr = aptr;
csconv = NULL;
ckeyl = 0;
ckey = NULL;
ctryl = 0;
ctry = NULL;
utf8 = 0;
langnum = 0;
complexprefixes = 0;
maxSug = maxn;
nosplitsugs = 0;
maxngramsugs = MAXNGRAMSUGS;
maxcpdsugs = MAXCOMPOUNDSUGS;
if (pAMgr) {
langnum = pAMgr->get_langnum();
ckey = pAMgr->get_key_string();
nosplitsugs = pAMgr->get_nosplitsugs();
if (pAMgr->get_maxngramsugs() >= 0)
maxngramsugs = pAMgr->get_maxngramsugs();
utf8 = pAMgr->get_utf8();
if (pAMgr->get_maxcpdsugs() >= 0)
maxcpdsugs = pAMgr->get_maxcpdsugs();
if (!utf8) {
csconv = get_current_cs(pAMgr->get_encoding());
}
complexprefixes = pAMgr->get_complexprefixes();
}
if (ckey) {
if (utf8) {
ckeyl = u8_u16(ckey_utf, ckey);
} else {
ckeyl = strlen(ckey);
}
}
if (tryme) {
ctry = mystrdup(tryme);
if (ctry)
ctryl = strlen(ctry);
if (ctry && utf8) {
ctryl = u8_u16(ctry_utf, tryme);
}
}
// language with possible dash usage
// (latin letters or dash in TRY characters)
lang_with_dash_usage = (ctry &&
((strchr(ctry, '-') != NULL) || (strchr(ctry, 'a') != NULL)));
}
SuggestMgr::~SuggestMgr() {
pAMgr = NULL;
if (ckey)
free(ckey);
ckey = NULL;
ckeyl = 0;
if (ctry)
free(ctry);
ctry = NULL;
ctryl = 0;
maxSug = 0;
#ifdef MOZILLA_CLIENT
delete[] csconv;
#endif
}
void SuggestMgr::testsug(std::vector<std::string>& wlst,
const std::string& candidate,
int cpdsuggest,
int* timer,
clock_t* timelimit) {
int cwrd = 1;
if (wlst.size() == maxSug)
return;
for (size_t k = 0; k < wlst.size(); ++k) {
if (wlst[k] == candidate) {
cwrd = 0;
break;
}
}
if ((cwrd) && checkword(candidate, cpdsuggest, timer, timelimit)) {
wlst.push_back(candidate);
}
}
/* generate suggestions for a misspelled word
* pass in address of array of char * pointers
* onlycompoundsug: probably bad suggestions (need for ngram sugs, too)
* return value: true, if there is a good suggestion
* (REP, ph: or a dictionary word pair)
*/
bool SuggestMgr::suggest(std::vector<std::string>& slst,
const char* w,
int* onlycompoundsug) {
int nocompoundtwowords = 0;
std::vector<w_char> word_utf;
int wl = 0;
size_t nsugorig = slst.size();
std::string w2;
const char* word = w;
size_t oldSug = 0;
bool good_suggestion = false;
// word reversing wrapper for complex prefixes
if (complexprefixes) {
w2.assign(w);
if (utf8)
reverseword_utf(w2);
else
reverseword(w2);
word = w2.c_str();
}
if (utf8) {
wl = u8_u16(word_utf, word);
if (wl == -1) {
return false;
}
}
for (int cpdsuggest = 0; (cpdsuggest < 2) && (nocompoundtwowords == 0) && !good_suggestion;
cpdsuggest++) {
clock_t timelimit;
// initialize both in non-compound and compound cycles
timelimit = clock();
// limit compound suggestion
if (cpdsuggest > 0)
oldSug = slst.size();
// suggestions for an uppercase word (html -> HTML)
if (slst.size() < maxSug) {
size_t i = slst.size();
if (utf8)
capchars_utf(slst, &word_utf[0], wl, cpdsuggest);
else
capchars(slst, word, cpdsuggest);
if (slst.size() > i)
good_suggestion = true;
}
// perhaps we made a typical fault of spelling
if ((slst.size() < maxSug) && (!cpdsuggest || (slst.size() < oldSug + maxcpdsugs))) {
size_t i = slst.size();
replchars(slst, word, cpdsuggest);
if (slst.size() > i)
good_suggestion = true;
}
if (clock() > timelimit + TIMELIMIT_SUGGESTION)
return good_suggestion;
// perhaps we made chose the wrong char from a related set
if ((slst.size() < maxSug) &&
(!cpdsuggest || (slst.size() < oldSug + maxcpdsugs))) {
mapchars(slst, word, cpdsuggest);
}
if (clock() > timelimit + TIMELIMIT_SUGGESTION)
return good_suggestion;
// only suggest compound words when no other suggestion
if ((cpdsuggest == 0) && (slst.size() > nsugorig))
nocompoundtwowords = 1;
// did we swap the order of chars by mistake
if ((slst.size() < maxSug) && (!cpdsuggest || (slst.size() < oldSug + maxcpdsugs))) {
if (utf8)
swapchar_utf(slst, &word_utf[0], wl, cpdsuggest);
else
swapchar(slst, word, cpdsuggest);
}
if (clock() > timelimit + TIMELIMIT_SUGGESTION)
return good_suggestion;
// did we swap the order of non adjacent chars by mistake
if ((slst.size() < maxSug) && (!cpdsuggest || (slst.size() < oldSug + maxcpdsugs))) {
if (utf8)
longswapchar_utf(slst, &word_utf[0], wl, cpdsuggest);
else
longswapchar(slst, word, cpdsuggest);
}
if (clock() > timelimit + TIMELIMIT_SUGGESTION)
return good_suggestion;
// did we just hit the wrong key in place of a good char (case and keyboard)
if ((slst.size() < maxSug) && (!cpdsuggest || (slst.size() < oldSug + maxcpdsugs))) {
if (utf8)
badcharkey_utf(slst, &word_utf[0], wl, cpdsuggest);
else
badcharkey(slst, word, cpdsuggest);
}
if (clock() > timelimit + TIMELIMIT_SUGGESTION)
return good_suggestion;
// did we add a char that should not be there
if ((slst.size() < maxSug) && (!cpdsuggest || (slst.size() < oldSug + maxcpdsugs))) {
if (utf8)
extrachar_utf(slst, &word_utf[0], wl, cpdsuggest);
else
extrachar(slst, word, cpdsuggest);
}
if (clock() > timelimit + TIMELIMIT_SUGGESTION)
return good_suggestion;
// did we forgot a char
if ((slst.size() < maxSug) && (!cpdsuggest || (slst.size() < oldSug + maxcpdsugs))) {
if (utf8)
forgotchar_utf(slst, &word_utf[0], wl, cpdsuggest);
else
forgotchar(slst, word, cpdsuggest);
}
if (clock() > timelimit + TIMELIMIT_SUGGESTION)
return good_suggestion;
// did we move a char
if ((slst.size() < maxSug) && (!cpdsuggest || (slst.size() < oldSug + maxcpdsugs))) {
if (utf8)
movechar_utf(slst, &word_utf[0], wl, cpdsuggest);
else
movechar(slst, word, cpdsuggest);
}
if (clock() > timelimit + TIMELIMIT_SUGGESTION)
return good_suggestion;
// did we just hit the wrong key in place of a good char
if ((slst.size() < maxSug) && (!cpdsuggest || (slst.size() < oldSug + maxcpdsugs))) {
if (utf8)
badchar_utf(slst, &word_utf[0], wl, cpdsuggest);
else
badchar(slst, word, cpdsuggest);
}
if (clock() > timelimit + TIMELIMIT_SUGGESTION)
return good_suggestion;
// did we double two characters
if ((slst.size() < maxSug) && (!cpdsuggest || (slst.size() < oldSug + maxcpdsugs))) {
if (utf8)
doubletwochars_utf(slst, &word_utf[0], wl, cpdsuggest);
else
doubletwochars(slst, word, cpdsuggest);
}
if (clock() > timelimit + TIMELIMIT_SUGGESTION)
return good_suggestion;
// perhaps we forgot to hit space and two words ran together
// (dictionary word pairs have top priority here, so
// we always suggest them, in despite of nosplitsugs, and
// drop compound word and other suggestions)
if (!cpdsuggest || (!nosplitsugs && slst.size() < oldSug + maxcpdsugs)) {
good_suggestion = twowords(slst, word, cpdsuggest, good_suggestion);
}
if (clock() > timelimit + TIMELIMIT_SUGGESTION)
return good_suggestion;
} // repeating ``for'' statement compounding support
if (!nocompoundtwowords && (!slst.empty()) && onlycompoundsug)
*onlycompoundsug = 1;
return good_suggestion;
}
// suggestions for an uppercase word (html -> HTML)
void SuggestMgr::capchars_utf(std::vector<std::string>& wlst,
const w_char* word,
int wl,
int cpdsuggest) {
std::vector<w_char> candidate_utf(word, word + wl);
mkallcap_utf(candidate_utf, langnum);
std::string candidate;
u16_u8(candidate, candidate_utf);
testsug(wlst, candidate, cpdsuggest, NULL, NULL);
}
// suggestions for an uppercase word (html -> HTML)
void SuggestMgr::capchars(std::vector<std::string>& wlst,
const char* word,
int cpdsuggest) {
std::string candidate(word);
mkallcap(candidate, csconv);
testsug(wlst, candidate, cpdsuggest, NULL, NULL);
}
// suggestions for when chose the wrong char out of a related set
int SuggestMgr::mapchars(std::vector<std::string>& wlst,
const char* word,
int cpdsuggest) {
std::string candidate;
clock_t timelimit;
int timer;
int wl = strlen(word);
if (wl < 2 || !pAMgr)
return wlst.size();
const std::vector<mapentry>& maptable = pAMgr->get_maptable();
if (maptable.empty())
return wlst.size();
timelimit = clock();
timer = MINTIMER;
return map_related(word, candidate, 0, wlst, cpdsuggest,
maptable, &timer, &timelimit);
}
int SuggestMgr::map_related(const char* word,
std::string& candidate,
int wn,
std::vector<std::string>& wlst,
int cpdsuggest,
const std::vector<mapentry>& maptable,
int* timer,
clock_t* timelimit) {
if (*(word + wn) == '\0') {
int cwrd = 1;
for (size_t m = 0; m < wlst.size(); ++m) {
if (wlst[m] == candidate) {
cwrd = 0;
break;
}
}
if ((cwrd) && checkword(candidate, cpdsuggest, timer, timelimit)) {
if (wlst.size() < maxSug) {
wlst.push_back(candidate);
}
}
return wlst.size();
}
int in_map = 0;
for (size_t j = 0; j < maptable.size(); ++j) {
for (size_t k = 0; k < maptable[j].size(); ++k) {
size_t len = maptable[j][k].size();
if (strncmp(maptable[j][k].c_str(), word + wn, len) == 0) {
in_map = 1;
size_t cn = candidate.size();
for (size_t l = 0; l < maptable[j].size(); ++l) {
candidate.resize(cn);
candidate.append(maptable[j][l]);
map_related(word, candidate, wn + len, wlst,
cpdsuggest, maptable, timer, timelimit);
if (!(*timer))
return wlst.size();
}
}
}
}
if (!in_map) {
candidate.push_back(*(word + wn));
map_related(word, candidate, wn + 1, wlst, cpdsuggest,
maptable, timer, timelimit);
}
return wlst.size();
}
// suggestions for a typical fault of spelling, that
// differs with more, than 1 letter from the right form.
int SuggestMgr::replchars(std::vector<std::string>& wlst,
const char* word,
int cpdsuggest) {
std::string candidate;
int wl = strlen(word);
if (wl < 2 || !pAMgr)
return wlst.size();
const std::vector<replentry>& reptable = pAMgr->get_reptable();
for (size_t i = 0; i < reptable.size(); ++i) {
const char* r = word;
// search every occurence of the pattern in the word
while ((r = strstr(r, reptable[i].pattern.c_str())) != NULL) {
int type = (r == word) ? 1 : 0;
if (r - word + reptable[i].pattern.size() == strlen(word))
type += 2;
while (type && reptable[i].outstrings[type].empty())
type = (type == 2 && r != word) ? 0 : type - 1;
const std::string&out = reptable[i].outstrings[type];
if (out.empty()) {
++r;
continue;
}
candidate.assign(word);
candidate.resize(r - word);
candidate.append(reptable[i].outstrings[type]);
candidate.append(r + reptable[i].pattern.size());
testsug(wlst, candidate, cpdsuggest, NULL, NULL);
// check REP suggestions with space
size_t sp = candidate.find(' ');
if (sp != std::string::npos) {
size_t prev = 0;
while (sp != std::string::npos) {
std::string prev_chunk = candidate.substr(prev, sp - prev);
if (checkword(prev_chunk, 0, NULL, NULL)) {
size_t oldns = wlst.size();
std::string post_chunk = candidate.substr(sp + 1);
testsug(wlst, post_chunk, cpdsuggest, NULL, NULL);
if (oldns < wlst.size()) {
wlst[wlst.size() - 1] = candidate;
}
}
prev = sp + 1;
sp = candidate.find(' ', prev);
}
}
r++; // search for the next letter
}
}
return wlst.size();
}
// perhaps we doubled two characters
// (for example vacation -> vacacation)
// The recognized pattern with regex back-references:
// "(.)(.)\1\2\1" or "..(.)(.)\1\2"
int SuggestMgr::doubletwochars(std::vector<std::string>& wlst,
const char* word,
int cpdsuggest) {
int state = 0;
int wl = strlen(word);
if (wl < 5 || !pAMgr)
return wlst.size();
for (int i = 2; i < wl; i++) {
if (word[i] == word[i - 2]) {
state++;
if (state == 3 || (state == 2 && i >= 4)) {
std::string candidate(word, word + i - 1);
candidate.insert(candidate.end(), word + i + 1, word + wl);
testsug(wlst, candidate, cpdsuggest, NULL, NULL);
state = 0;
}
} else {
state = 0;
}
}
return wlst.size();
}
// perhaps we doubled two characters
// (for example vacation -> vacacation)
// The recognized pattern with regex back-references:
// "(.)(.)\1\2\1" or "..(.)(.)\1\2"
int SuggestMgr::doubletwochars_utf(std::vector<std::string>& wlst,
const w_char* word,
int wl,
int cpdsuggest) {
int state = 0;
if (wl < 5 || !pAMgr)
return wlst.size();
for (int i = 2; i < wl; i++) {
if (word[i] == word[i - 2]) {
state++;
if (state == 3 || (state == 2 && i >= 4)) {
std::vector<w_char> candidate_utf(word, word + i - 1);
candidate_utf.insert(candidate_utf.end(), word + i + 1, word + wl);
std::string candidate;
u16_u8(candidate, candidate_utf);
testsug(wlst, candidate, cpdsuggest, NULL, NULL);
state = 0;
}
} else {
state = 0;
}
}
return wlst.size();
}
// error is wrong char in place of correct one (case and keyboard related
// version)
int SuggestMgr::badcharkey(std::vector<std::string>& wlst,
const char* word,
int cpdsuggest) {
std::string candidate(word);
// swap out each char one by one and try uppercase and neighbor
// keyboard chars in its place to see if that makes a good word
for (size_t i = 0; i < candidate.size(); ++i) {
char tmpc = candidate[i];
// check with uppercase letters
candidate[i] = csconv[((unsigned char)tmpc)].cupper;
if (tmpc != candidate[i]) {
testsug(wlst, candidate, cpdsuggest, NULL, NULL);
candidate[i] = tmpc;
}
// check neighbor characters in keyboard string
if (!ckey)
continue;
char* loc = strchr(ckey, tmpc);
while (loc) {
if ((loc > ckey) && (*(loc - 1) != '|')) {
candidate[i] = *(loc - 1);
testsug(wlst, candidate, cpdsuggest, NULL, NULL);
}
if ((*(loc + 1) != '|') && (*(loc + 1) != '\0')) {
candidate[i] = *(loc + 1);
testsug(wlst, candidate, cpdsuggest, NULL, NULL);
}
loc = strchr(loc + 1, tmpc);
}
candidate[i] = tmpc;
}
return wlst.size();
}
// error is wrong char in place of correct one (case and keyboard related
// version)
int SuggestMgr::badcharkey_utf(std::vector<std::string>& wlst,
const w_char* word,
int wl,
int cpdsuggest) {
std::string candidate;
std::vector<w_char> candidate_utf(word, word + wl);
// swap out each char one by one and try all the tryme
// chars in its place to see if that makes a good word
for (int i = 0; i < wl; i++) {
w_char tmpc = candidate_utf[i];
// check with uppercase letters
candidate_utf[i] = upper_utf(candidate_utf[i], 1);
if (tmpc != candidate_utf[i]) {
u16_u8(candidate, candidate_utf);
testsug(wlst, candidate, cpdsuggest, NULL, NULL);
candidate_utf[i] = tmpc;
}
// check neighbor characters in keyboard string
if (!ckey)
continue;
size_t loc = 0;
while ((loc < ckeyl) && ckey_utf[loc] != tmpc)
++loc;
while (loc < ckeyl) {
if ((loc > 0) && ckey_utf[loc - 1] != W_VLINE) {
candidate_utf[i] = ckey_utf[loc - 1];
u16_u8(candidate, candidate_utf);
testsug(wlst, candidate, cpdsuggest, NULL, NULL);
}
if (((loc + 1) < ckeyl) && (ckey_utf[loc + 1] != W_VLINE)) {
candidate_utf[i] = ckey_utf[loc + 1];
u16_u8(candidate, candidate_utf);
testsug(wlst, candidate, cpdsuggest, NULL, NULL);
}
do {
loc++;
} while ((loc < ckeyl) && ckey_utf[loc] != tmpc);
}
candidate_utf[i] = tmpc;
}
return wlst.size();
}
// error is wrong char in place of correct one
int SuggestMgr::badchar(std::vector<std::string>& wlst, const char* word, int cpdsuggest) {
std::string candidate(word);
clock_t timelimit = clock();
int timer = MINTIMER;
// swap out each char one by one and try all the tryme
// chars in its place to see if that makes a good word
for (size_t j = 0; j < ctryl; ++j) {
for (std::string::reverse_iterator aI = candidate.rbegin(), aEnd = candidate.rend(); aI != aEnd; ++aI) {
char tmpc = *aI;
if (ctry[j] == tmpc)
continue;
*aI = ctry[j];
testsug(wlst, candidate, cpdsuggest, &timer, &timelimit);
if (!timer)
return wlst.size();
*aI = tmpc;
}
}
return wlst.size();
}
// error is wrong char in place of correct one
int SuggestMgr::badchar_utf(std::vector<std::string>& wlst,
const w_char* word,
int wl,
int cpdsuggest) {
std::vector<w_char> candidate_utf(word, word + wl);
std::string candidate;
clock_t timelimit = clock();
int timer = MINTIMER;
// swap out each char one by one and try all the tryme
// chars in its place to see if that makes a good word
for (size_t j = 0; j < ctryl; ++j) {
for (int i = wl - 1; i >= 0; i--) {
w_char tmpc = candidate_utf[i];
if (tmpc == ctry_utf[j])
continue;
candidate_utf[i] = ctry_utf[j];
u16_u8(candidate, candidate_utf);
testsug(wlst, candidate, cpdsuggest, &timer, &timelimit);
if (!timer)
return wlst.size();
candidate_utf[i] = tmpc;
}
}
return wlst.size();
}
// error is word has an extra letter it does not need
int SuggestMgr::extrachar_utf(std::vector<std::string>& wlst,
const w_char* word,
int wl,
int cpdsuggest) {
std::vector<w_char> candidate_utf(word, word + wl);
if (candidate_utf.size() < 2)
return wlst.size();
// try omitting one char of word at a time
for (size_t i = 0; i < candidate_utf.size(); ++i) {
size_t index = candidate_utf.size() - 1 - i;
w_char tmpc = candidate_utf[index];
candidate_utf.erase(candidate_utf.begin() + index);
std::string candidate;
u16_u8(candidate, candidate_utf);
testsug(wlst, candidate, cpdsuggest, NULL, NULL);
candidate_utf.insert(candidate_utf.begin() + index, tmpc);
}
return wlst.size();
}
// error is word has an extra letter it does not need
int SuggestMgr::extrachar(std::vector<std::string>& wlst,
const char* word,
int cpdsuggest) {
std::string candidate(word);
if (candidate.size() < 2)
return wlst.size();
// try omitting one char of word at a time
for (size_t i = 0; i < candidate.size(); ++i) {
size_t index = candidate.size() - 1 - i;
char tmpc = candidate[index];
candidate.erase(candidate.begin() + index);
testsug(wlst, candidate, cpdsuggest, NULL, NULL);
candidate.insert(candidate.begin() + index, tmpc);
}
return wlst.size();
}
// error is missing a letter it needs
int SuggestMgr::forgotchar(std::vector<std::string>& wlst,
const char* word,
int cpdsuggest) {
std::string candidate(word);
clock_t timelimit = clock();
int timer = MINTIMER;
// try inserting a tryme character before every letter (and the null
// terminator)
for (size_t k = 0; k < ctryl; ++k) {
for (size_t i = 0; i <= candidate.size(); ++i) {
size_t index = candidate.size() - i;
candidate.insert(candidate.begin() + index, ctry[k]);
testsug(wlst, candidate, cpdsuggest, &timer, &timelimit);
if (!timer)
return wlst.size();
candidate.erase(candidate.begin() + index);
}
}
return wlst.size();
}
// error is missing a letter it needs
int SuggestMgr::forgotchar_utf(std::vector<std::string>& wlst,
const w_char* word,
int wl,
int cpdsuggest) {
std::vector<w_char> candidate_utf(word, word + wl);
clock_t timelimit = clock();
int timer = MINTIMER;
// try inserting a tryme character at the end of the word and before every
// letter
for (size_t k = 0; k < ctryl; ++k) {
for (size_t i = 0; i <= candidate_utf.size(); ++i) {
size_t index = candidate_utf.size() - i;
candidate_utf.insert(candidate_utf.begin() + index, ctry_utf[k]);
std::string candidate;
u16_u8(candidate, candidate_utf);
testsug(wlst, candidate, cpdsuggest, &timer, &timelimit);
if (!timer)
return wlst.size();
candidate_utf.erase(candidate_utf.begin() + index);
}
}
return wlst.size();
}
/* error is should have been two words
* return value is true, if there is a dictionary word pair,
* or there was already a good suggestion before calling
* this function.
*/
bool SuggestMgr::twowords(std::vector<std::string>& wlst,
const char* word,
int cpdsuggest,
bool good) {
int c2;
int forbidden = 0;
int cwrd;
int wl = strlen(word);
if (wl < 3)
return false;
if (langnum == LANG_hu)
forbidden = check_forbidden(word, wl);
char* candidate = (char*)malloc(wl + 2);
strcpy(candidate + 1, word);
// split the string into two pieces after every char
// if both pieces are good words make them a suggestion
for (char* p = candidate + 1; p[1] != '\0'; p++) {
p[-1] = *p;
// go to end of the UTF-8 character
while (utf8 && ((p[1] & 0xc0) == 0x80)) {
*p = p[1];
p++;
}
if (utf8 && p[1] == '\0')
break; // last UTF-8 character
// Suggest only word pairs, if they are listed in the dictionary.
// For example, adding "a lot" to the English dic file will
// result only "alot" -> "a lot" suggestion instead of
// "alto, slot, alt, lot, allot, aloft, aloe, clot, plot, blot, a lot".
// Note: using "ph:alot" keeps the other suggestions:
// a lot ph:alot
// alot -> a lot, alto, slot...
*p = ' ';
if (!cpdsuggest && checkword(candidate, cpdsuggest, NULL, NULL)) {
// remove not word pair suggestions
if (!good) {
good = true;
wlst.clear();
}
wlst.insert(wlst.begin(), candidate);
}
// word pairs with dash?
if (lang_with_dash_usage) {
*p = '-';
if (!cpdsuggest && checkword(candidate, cpdsuggest, NULL, NULL)) {
// remove not word pair suggestions
if (!good) {
good = true;
wlst.clear();
}
wlst.insert(wlst.begin(), candidate);
}
}
if (wlst.size() < maxSug && !nosplitsugs && !good) {
*p = '\0';
int c1 = checkword(candidate, cpdsuggest, NULL, NULL);
if (c1) {
c2 = checkword((p + 1), cpdsuggest, NULL, NULL);
if (c2) {
// spec. Hungarian code (TODO need a better compound word support)
if ((langnum == LANG_hu) && !forbidden &&
// if 3 repeating letter, use - instead of space
(((p[-1] == p[1]) &&
(((p > candidate + 1) && (p[-1] == p[-2])) || (p[-1] == p[2]))) ||
// or multiple compounding, with more, than 6 syllables
((c1 == 3) && (c2 >= 2))))
*p = '-';
else
*p = ' ';
cwrd = 1;
for (size_t k = 0; k < wlst.size(); ++k) {
if (wlst[k] == candidate) {
cwrd = 0;
break;
}
}
if (cwrd && (wlst.size() < maxSug))
wlst.push_back(candidate);
// add two word suggestion with dash, depending on the language
// Note that cwrd doesn't modified for REP twoword sugg.
if ( !nosplitsugs && lang_with_dash_usage &&
mystrlen(p + 1) > 1 && mystrlen(candidate) - mystrlen(p) > 1) {
*p = '-';
for (size_t k = 0; k < wlst.size(); ++k) {
if (wlst[k] == candidate) {
cwrd = 0;
break;
}
}
if ((wlst.size() < maxSug) && cwrd)
wlst.push_back(candidate);
}
}
}
}
}
free(candidate);
return good;
}
// error is adjacent letter were swapped
int SuggestMgr::swapchar(std::vector<std::string>& wlst,
const char* word,
int cpdsuggest) {
std::string candidate(word);
if (candidate.size() < 2)
return wlst.size();
// try swapping adjacent chars one by one
for (size_t i = 0; i < candidate.size() - 1; ++i) {
std::swap(candidate[i], candidate[i+1]);
testsug(wlst, candidate, cpdsuggest, NULL, NULL);
std::swap(candidate[i], candidate[i+1]);
}
// try double swaps for short words
// ahev -> have, owudl -> would
if (candidate.size() == 4 || candidate.size() == 5) {
candidate[0] = word[1];
candidate[1] = word[0];
candidate[2] = word[2];
candidate[candidate.size() - 2] = word[candidate.size() - 1];
candidate[candidate.size() - 1] = word[candidate.size() - 2];
testsug(wlst, candidate, cpdsuggest, NULL, NULL);
if (candidate.size() == 5) {
candidate[0] = word[0];
candidate[1] = word[2];
candidate[2] = word[1];
testsug(wlst, candidate, cpdsuggest, NULL, NULL);
}
}
return wlst.size();
}
// error is adjacent letter were swapped
int SuggestMgr::swapchar_utf(std::vector<std::string>& wlst,
const w_char* word,
int wl,
int cpdsuggest) {
std::vector<w_char> candidate_utf(word, word + wl);
if (candidate_utf.size() < 2)
return wlst.size();
std::string candidate;
// try swapping adjacent chars one by one
for (size_t i = 0; i < candidate_utf.size() - 1; ++i) {
std::swap(candidate_utf[i], candidate_utf[i+1]);
u16_u8(candidate, candidate_utf);
testsug(wlst, candidate, cpdsuggest, NULL, NULL);
std::swap(candidate_utf[i], candidate_utf[i+1]);
}
// try double swaps for short words
// ahev -> have, owudl -> would, suodn -> sound
if (candidate_utf.size() == 4 || candidate_utf.size() == 5) {
candidate_utf[0] = word[1];
candidate_utf[1] = word[0];
candidate_utf[2] = word[2];
candidate_utf[candidate_utf.size() - 2] = word[candidate_utf.size() - 1];
candidate_utf[candidate_utf.size() - 1] = word[candidate_utf.size() - 2];
u16_u8(candidate, candidate_utf);
testsug(wlst, candidate, cpdsuggest, NULL, NULL);
if (candidate_utf.size() == 5) {
candidate_utf[0] = word[0];
candidate_utf[1] = word[2];
candidate_utf[2] = word[1];
u16_u8(candidate, candidate_utf);
testsug(wlst, candidate, cpdsuggest, NULL, NULL);
}
}
return wlst.size();
}
// error is not adjacent letter were swapped
int SuggestMgr::longswapchar(std::vector<std::string>& wlst,
const char* word,
int cpdsuggest) {
std::string candidate(word);
// try swapping not adjacent chars one by one
for (std::string::iterator p = candidate.begin(); p < candidate.end(); ++p) {
for (std::string::iterator q = candidate.begin(); q < candidate.end(); ++q) {
size_t distance = std::abs(std::distance(q, p));
if (distance > 1 && distance <= MAX_CHAR_DISTANCE) {
std::swap(*p, *q);
testsug(wlst, candidate, cpdsuggest, NULL, NULL);
std::swap(*p, *q);
}
}
}
return wlst.size();
}
// error is adjacent letter were swapped
int SuggestMgr::longswapchar_utf(std::vector<std::string>& wlst,
const w_char* word,
int wl,
int cpdsuggest) {
std::vector<w_char> candidate_utf(word, word + wl);
// try swapping not adjacent chars
for (std::vector<w_char>::iterator p = candidate_utf.begin(); p < candidate_utf.end(); ++p) {
for (std::vector<w_char>::iterator q = candidate_utf.begin(); q < candidate_utf.end(); ++q) {
size_t distance = std::abs(std::distance(q, p));
if (distance > 1 && distance <= MAX_CHAR_DISTANCE) {
std::swap(*p, *q);
std::string candidate;
u16_u8(candidate, candidate_utf);
testsug(wlst, candidate, cpdsuggest, NULL, NULL);
std::swap(*p, *q);
}
}
}
return wlst.size();
}
// error is a letter was moved
int SuggestMgr::movechar(std::vector<std::string>& wlst,
const char* word,
int cpdsuggest) {
std::string candidate(word);
if (candidate.size() < 2)
return wlst.size();
// try moving a char
for (std::string::iterator p = candidate.begin(); p < candidate.end(); ++p) {
for (std::string::iterator q = p + 1; q < candidate.end() && std::distance(p, q) <= MAX_CHAR_DISTANCE; ++q) {
std::swap(*q, *(q - 1));
if (std::distance(p, q) < 2)
continue; // omit swap char
testsug(wlst, candidate, cpdsuggest, NULL, NULL);
}
std::copy(word, word + candidate.size(), candidate.begin());
}
for (std::string::reverse_iterator p = candidate.rbegin(), pEnd = candidate.rend() - 1; p != pEnd; ++p) {
for (std::string::reverse_iterator q = p + 1, qEnd = candidate.rend(); q != qEnd && std::distance(p, q) <= MAX_CHAR_DISTANCE; ++q) {
std::swap(*q, *(q - 1));
if (std::distance(p, q) < 2)
continue; // omit swap char
testsug(wlst, candidate, cpdsuggest, NULL, NULL);
}
std::copy(word, word + candidate.size(), candidate.begin());
}
return wlst.size();
}
// error is a letter was moved
int SuggestMgr::movechar_utf(std::vector<std::string>& wlst,
const w_char* word,
int wl,
int cpdsuggest) {
std::vector<w_char> candidate_utf(word, word + wl);
if (candidate_utf.size() < 2)
return wlst.size();
// try moving a char
for (std::vector<w_char>::iterator p = candidate_utf.begin(); p < candidate_utf.end(); ++p) {
for (std::vector<w_char>::iterator q = p + 1; q < candidate_utf.end() && std::distance(p, q) <= MAX_CHAR_DISTANCE; ++q) {
std::swap(*q, *(q - 1));
if (std::distance(p, q) < 2)
continue; // omit swap char
std::string candidate;
u16_u8(candidate, candidate_utf);
testsug(wlst, candidate, cpdsuggest, NULL, NULL);
}
std::copy(word, word + candidate_utf.size(), candidate_utf.begin());
}
for (std::vector<w_char>::reverse_iterator p = candidate_utf.rbegin(); p < candidate_utf.rend(); ++p) {
for (std::vector<w_char>::reverse_iterator q = p + 1; q < candidate_utf.rend() && std::distance(p, q) <= MAX_CHAR_DISTANCE; ++q) {
std::swap(*q, *(q - 1));
if (std::distance(p, q) < 2)
continue; // omit swap char
std::string candidate;
u16_u8(candidate, candidate_utf);
testsug(wlst, candidate, cpdsuggest, NULL, NULL);
}
std::copy(word, word + candidate_utf.size(), candidate_utf.begin());
}
return wlst.size();
}
// generate a set of suggestions for very poorly spelled words
void SuggestMgr::ngsuggest(std::vector<std::string>& wlst,
const char* w,
const std::vector<HashMgr*>& rHMgr,
int captype) {
int lval;
int sc;
int lp, lpphon;
int nonbmp = 0;
// exhaustively search through all root words
// keeping track of the MAX_ROOTS most similar root words
struct hentry* roots[MAX_ROOTS];
char* rootsphon[MAX_ROOTS];
int scores[MAX_ROOTS];
int scoresphon[MAX_ROOTS];
for (int i = 0; i < MAX_ROOTS; i++) {
roots[i] = NULL;
scores[i] = -100 * i;
rootsphon[i] = NULL;
scoresphon[i] = -100 * i;
}
lp = MAX_ROOTS - 1;
lpphon = MAX_ROOTS - 1;
int low = NGRAM_LOWERING;
std::string w2;
const char* word = w;
// word reversing wrapper for complex prefixes
if (complexprefixes) {
w2.assign(w);
if (utf8)
reverseword_utf(w2);
else
reverseword(w2);
word = w2.c_str();
}
std::vector<w_char> u8;
int nc = strlen(word);
int n = (utf8) ? u8_u16(u8, word) : nc;
// set character based ngram suggestion for words with non-BMP Unicode
// characters
if (n == -1) {
utf8 = 0; // XXX not state-free
n = nc;
nonbmp = 1;
low = 0;
}
struct hentry* hp = NULL;
int col = -1;
phonetable* ph = (pAMgr) ? pAMgr->get_phonetable() : NULL;
std::string target;
std::string candidate;
std::vector<w_char> w_candidate;
if (ph) {
if (utf8) {
u8_u16(w_candidate, word);
mkallcap_utf(w_candidate, langnum);
u16_u8(candidate, w_candidate);
} else {
candidate.assign(word);
if (!nonbmp)
mkallcap(candidate, csconv);
}
target = phonet(candidate, *ph); // XXX phonet() is 8-bit (nc, not n)
}
FLAG forbiddenword = pAMgr ? pAMgr->get_forbiddenword() : FLAG_NULL;
FLAG nosuggest = pAMgr ? pAMgr->get_nosuggest() : FLAG_NULL;
FLAG nongramsuggest = pAMgr ? pAMgr->get_nongramsuggest() : FLAG_NULL;
FLAG onlyincompound = pAMgr ? pAMgr->get_onlyincompound() : FLAG_NULL;
std::vector<w_char> w_word, w_target;
if (utf8) {
u8_u16(w_word, word);
u8_u16(w_target, target);
}
std::string f;
std::vector<w_char> w_f;
for (size_t i = 0; i < rHMgr.size(); ++i) {
while (0 != (hp = rHMgr[i]->walk_hashtable(col, hp))) {
// skip exceptions
if (
// skip it, if the word length different by 5 or
// more characters (to avoid strange suggestions)
// (except Unicode characters over BMP)
(((abs(n - hp->clen) > 4) && !nonbmp)) ||
// don't suggest capitalized dictionary words for
// lower case misspellings in ngram suggestions, except
// - PHONE usage, or
// - in the case of German, where not only proper
// nouns are capitalized, or
// - the capitalized word has special pronunciation
((captype == NOCAP) && (hp->var & H_OPT_INITCAP) &&
!ph && (langnum != LANG_de) && !(hp->var & H_OPT_PHON)) ||
// or it has one of the following special flags
((hp->astr) && (pAMgr) &&
(TESTAFF(hp->astr, forbiddenword, hp->alen) ||
TESTAFF(hp->astr, ONLYUPCASEFLAG, hp->alen) ||
TESTAFF(hp->astr, nosuggest, hp->alen) ||
TESTAFF(hp->astr, nongramsuggest, hp->alen) ||
TESTAFF(hp->astr, onlyincompound, hp->alen)))
)
continue;
if (utf8) {
u8_u16(w_f, HENTRY_WORD(hp));
int leftcommon = leftcommonsubstring(w_word, w_f);
if (low) {
// lowering dictionary word
mkallsmall_utf(w_f, langnum);
}
sc = ngram(3, w_word, w_f, NGRAM_LONGER_WORSE) + leftcommon;
} else {
f.assign(HENTRY_WORD(hp));
int leftcommon = leftcommonsubstring(word, f.c_str());
if (low) {
// lowering dictionary word
mkallsmall(f, csconv);
}
sc = ngram(3, word, f, NGRAM_LONGER_WORSE) + leftcommon;
}
// check special pronunciation
f.clear();
if ((hp->var & H_OPT_PHON) &&
copy_field(f, HENTRY_DATA(hp), MORPH_PHON)) {
int sc2;
if (utf8) {
u8_u16(w_f, f);
int leftcommon = leftcommonsubstring(w_word, w_f);
if (low) {
// lowering dictionary word
mkallsmall_utf(w_f, langnum);
}
sc2 = ngram(3, w_word, w_f, NGRAM_LONGER_WORSE) + leftcommon;
} else {
int leftcommon = leftcommonsubstring(word, f.c_str());
if (low) {
// lowering dictionary word
mkallsmall(f, csconv);
}
sc2 = ngram(3, word, f, NGRAM_LONGER_WORSE) + leftcommon;
}
if (sc2 > sc)
sc = sc2;
}
int scphon = -20000;
if (ph && (sc > 2) && (abs(n - (int)hp->clen) <= 3)) {
if (utf8) {
u8_u16(w_candidate, HENTRY_WORD(hp));
mkallcap_utf(w_candidate, langnum);
u16_u8(candidate, w_candidate);
} else {
candidate = HENTRY_WORD(hp);
mkallcap(candidate, csconv);
}
f = phonet(candidate, *ph);
if (utf8) {
u8_u16(w_f, f);
scphon = 2 * ngram(3, w_target, w_f,
NGRAM_LONGER_WORSE);
} else {
scphon = 2 * ngram(3, target, f,
NGRAM_LONGER_WORSE);
}
}
if (sc > scores[lp]) {
scores[lp] = sc;
roots[lp] = hp;
lval = sc;
for (int j = 0; j < MAX_ROOTS; j++)
if (scores[j] < lval) {
lp = j;
lval = scores[j];
}
}
if (scphon > scoresphon[lpphon]) {
scoresphon[lpphon] = scphon;
rootsphon[lpphon] = HENTRY_WORD(hp);
lval = scphon;
for (int j = 0; j < MAX_ROOTS; j++)
if (scoresphon[j] < lval) {
lpphon = j;
lval = scoresphon[j];
}
}
}
}
// find minimum threshold for a passable suggestion
// mangle original word three differnt ways
// and score them to generate a minimum acceptable score
std::vector<w_char> w_mw;
int thresh = 0;
for (int sp = 1; sp < 4; sp++) {
if (utf8) {
w_mw = w_word;
for (int k = sp; k < n; k += 4) {
w_mw[k].l = '*';
w_mw[k].h = 0;
}
if (low) {
// lowering dictionary word
mkallsmall_utf(w_mw, langnum);
}
thresh += ngram(n, w_word, w_mw, NGRAM_ANY_MISMATCH);
} else {
std::string mw = word;
for (int k = sp; k < n; k += 4)
mw[k] = '*';
if (low) {
// lowering dictionary word
mkallsmall(mw, csconv);
}
thresh += ngram(n, word, mw, NGRAM_ANY_MISMATCH);
}
}
thresh = thresh / 3;
thresh--;
// now expand affixes on each of these root words and
// and use length adjusted ngram scores to select
// possible suggestions
char* guess[MAX_GUESS];
char* guessorig[MAX_GUESS];
int gscore[MAX_GUESS];
for (int i = 0; i < MAX_GUESS; i++) {
guess[i] = NULL;
guessorig[i] = NULL;
gscore[i] = -100 * i;
}
lp = MAX_GUESS - 1;
struct guessword* glst;
glst = (struct guessword*)calloc(MAX_WORDS, sizeof(struct guessword));
if (!glst) {
if (nonbmp)
utf8 = 1;
return;
}
for (int i = 0; i < MAX_ROOTS; i++) {
if (roots[i]) {
struct hentry* rp = roots[i];
f.clear();
const char *field = NULL;
if ((rp->var & H_OPT_PHON) && copy_field(f, HENTRY_DATA(rp), MORPH_PHON))
field = f.c_str();
int nw = pAMgr->expand_rootword(
glst, MAX_WORDS, HENTRY_WORD(rp), rp->blen, rp->astr, rp->alen, word,
nc, field);
for (int k = 0; k < nw; k++) {
if (utf8) {
u8_u16(w_f, glst[k].word);
int leftcommon = leftcommonsubstring(w_word, w_f);
if (low) {
// lowering dictionary word
mkallsmall_utf(w_f, langnum);
}
sc = ngram(n, w_word, w_f, NGRAM_ANY_MISMATCH) + leftcommon;
} else {
f = glst[k].word;
int leftcommon = leftcommonsubstring(word, f.c_str());
if (low) {
// lowering dictionary word
mkallsmall(f, csconv);
}
sc = ngram(n, word, f, NGRAM_ANY_MISMATCH) + leftcommon;
}
if (sc > thresh) {
if (sc > gscore[lp]) {
if (guess[lp]) {
free(guess[lp]);
if (guessorig[lp]) {
free(guessorig[lp]);
guessorig[lp] = NULL;
}
}
gscore[lp] = sc;
guess[lp] = glst[k].word;
guessorig[lp] = glst[k].orig;
lval = sc;
for (int j = 0; j < MAX_GUESS; j++)
if (gscore[j] < lval) {
lp = j;
lval = gscore[j];
}
} else {
free(glst[k].word);
if (glst[k].orig)
free(glst[k].orig);
}
} else {
free(glst[k].word);
if (glst[k].orig)
free(glst[k].orig);
}
}
}
}
free(glst);
// now we are done generating guesses
// sort in order of decreasing score
bubblesort(&guess[0], &guessorig[0], &gscore[0], MAX_GUESS);
if (ph)
bubblesort(&rootsphon[0], NULL, &scoresphon[0], MAX_ROOTS);
// weight suggestions with a similarity index, based on
// the longest common subsequent algorithm and resort
int is_swap = 0;
int re = 0;
double fact = 1.0;
if (pAMgr) {
int maxd = pAMgr->get_maxdiff();
if (maxd >= 0)
fact = (10.0 - maxd) / 5.0;
}
std::vector<w_char> w_gl;
for (int i = 0; i < MAX_GUESS; i++) {
if (guess[i]) {
// lowering guess[i]
std::string gl;
int len;
if (utf8) {
len = u8_u16(w_gl, guess[i]);
mkallsmall_utf(w_gl, langnum);
u16_u8(gl, w_gl);
} else {
gl.assign(guess[i]);
if (!nonbmp)
mkallsmall(gl, csconv);
len = strlen(guess[i]);
}
int _lcs = lcslen(word, gl.c_str());
// same characters with different casing
if ((n == len) && (n == _lcs)) {
gscore[i] += 2000;
break;
}
// using 2-gram instead of 3, and other weightening
if (utf8) {
u8_u16(w_gl, gl);
//w_gl is lowercase already at this point
re = ngram(2, w_word, w_gl, NGRAM_ANY_MISMATCH + NGRAM_WEIGHTED);
if (low) {
w_f = w_word;
// lowering dictionary word
mkallsmall_utf(w_f, langnum);
re += ngram(2, w_gl, w_f, NGRAM_ANY_MISMATCH + NGRAM_WEIGHTED);
} else {
re += ngram(2, w_gl, w_word, NGRAM_ANY_MISMATCH + NGRAM_WEIGHTED);
}
} else {
//gl is lowercase already at this point
re = ngram(2, word, gl, NGRAM_ANY_MISMATCH + NGRAM_WEIGHTED);
if (low) {
f = word;
// lowering dictionary word
mkallsmall(f, csconv);
re += ngram(2, gl, f, NGRAM_ANY_MISMATCH + NGRAM_WEIGHTED);
} else {
re += ngram(2, gl, word, NGRAM_ANY_MISMATCH + NGRAM_WEIGHTED);
}
}
int ngram_score, leftcommon_score;
if (utf8) {
//w_gl is lowercase already at this point
ngram_score = ngram(4, w_word, w_gl, NGRAM_ANY_MISMATCH);
leftcommon_score = leftcommonsubstring(w_word, w_gl);
} else {
//gl is lowercase already at this point
ngram_score = ngram(4, word, gl, NGRAM_ANY_MISMATCH);
leftcommon_score = leftcommonsubstring(word, gl.c_str());
}
gscore[i] =
// length of longest common subsequent minus length difference
2 * _lcs - abs((int)(n - len)) +
// weight length of the left common substring
leftcommon_score +
// weight equal character positions
(!nonbmp && commoncharacterpositions(word, gl.c_str(), &is_swap)
? 1
: 0) +
// swap character (not neighboring)
((is_swap) ? 10 : 0) +
// ngram
ngram_score +
// weighted ngrams
re +
// different limit for dictionaries with PHONE rules
(ph ? (re < len * fact ? -1000 : 0)
: (re < (n + len) * fact ? -1000 : 0));
}
}
bubblesort(&guess[0], &guessorig[0], &gscore[0], MAX_GUESS);
// phonetic version
if (ph)
for (int i = 0; i < MAX_ROOTS; i++) {
if (rootsphon[i]) {
// lowering rootphon[i]
std::string gl;
int len;
if (utf8) {
len = u8_u16(w_gl, rootsphon[i]);
mkallsmall_utf(w_gl, langnum);
u16_u8(gl, w_gl);
} else {
gl.assign(rootsphon[i]);
if (!nonbmp)
mkallsmall(gl, csconv);
len = strlen(rootsphon[i]);
}
// weight length of the left common substring
int leftcommon_score;
if (utf8)
leftcommon_score = leftcommonsubstring(w_word, w_gl);
else
leftcommon_score = leftcommonsubstring(word, gl.c_str());
// heuristic weigthing of ngram scores
scoresphon[i] += 2 * lcslen(word, gl) - abs((int)(n - len)) +
leftcommon_score;
}
}
if (ph)
bubblesort(&rootsphon[0], NULL, &scoresphon[0], MAX_ROOTS);
// copy over
size_t oldns = wlst.size();
int same = 0;
for (int i = 0; i < MAX_GUESS; i++) {
if (guess[i]) {
if ((wlst.size() < oldns + maxngramsugs) && (wlst.size() < maxSug) &&
(!same || (gscore[i] > 1000))) {
int unique = 1;
// leave only excellent suggestions, if exists
if (gscore[i] > 1000)
same = 1;
else if (gscore[i] < -100) {
same = 1;
// keep the best ngram suggestions, unless in ONLYMAXDIFF mode
if (wlst.size() > oldns || (pAMgr && pAMgr->get_onlymaxdiff())) {
free(guess[i]);
if (guessorig[i])
free(guessorig[i]);
continue;
}
}
for (size_t j = 0; j < wlst.size(); ++j) {
// don't suggest previous suggestions or a previous suggestion with
// prefixes or affixes
if ((!guessorig[i] && strstr(guess[i], wlst[j].c_str())) ||
(guessorig[i] && strstr(guessorig[i], wlst[j].c_str())) ||
// check forbidden words
!checkword(guess[i], 0, NULL, NULL)) {
unique = 0;
break;
}
}
if (unique) {
if (guessorig[i]) {
wlst.push_back(guessorig[i]);
} else {
wlst.push_back(guess[i]);
}
}
free(guess[i]);
if (guessorig[i])
free(guessorig[i]);
} else {
free(guess[i]);
if (guessorig[i])
free(guessorig[i]);
}
}
}
oldns = wlst.size();
if (ph)
for (int i = 0; i < MAX_ROOTS; i++) {
if (rootsphon[i]) {
if ((wlst.size() < oldns + MAXPHONSUGS) && (wlst.size() < maxSug)) {
int unique = 1;
for (size_t j = 0; j < wlst.size(); ++j) {
// don't suggest previous suggestions or a previous suggestion with
// prefixes or affixes
if (strstr(rootsphon[i], wlst[j].c_str()) ||
// check forbidden words
!checkword(rootsphon[i], 0, NULL, NULL)) {
unique = 0;
break;
}
}
if (unique) {
wlst.push_back(rootsphon[i]);
}
}
}
}
if (nonbmp)
utf8 = 1;
}
// see if a candidate suggestion is spelled correctly
// needs to check both root words and words with affixes
// obsolote MySpell-HU modifications:
// return value 2 and 3 marks compounding with hyphen (-)
// `3' marks roots without suffix
int SuggestMgr::checkword(const std::string& word,
int cpdsuggest,
int* timer,
clock_t* timelimit) {
// check time limit
if (timer) {
(*timer)--;
if (!(*timer) && timelimit) {
if ((clock() - *timelimit) > TIMELIMIT)
return 0;
*timer = MAXPLUSTIMER;
}
}
if (pAMgr) {
struct hentry* rv = NULL;
int nosuffix = 0;
if (cpdsuggest == 1) {
if (pAMgr->get_compound()) {
struct hentry* rv2 = NULL;
struct hentry* rwords[100]; // buffer for COMPOUND pattern checking
rv = pAMgr->compound_check(word, 0, 0, 100, 0, NULL, (hentry**)&rwords, 0, 1, 0); // EXT
if (rv &&
(!(rv2 = pAMgr->lookup(word.c_str())) || !rv2->astr ||
!(TESTAFF(rv2->astr, pAMgr->get_forbiddenword(), rv2->alen) ||
TESTAFF(rv2->astr, pAMgr->get_nosuggest(), rv2->alen))))
return 3; // XXX obsolote categorisation + only ICONV needs affix
// flag check?
}
return 0;
}
rv = pAMgr->lookup(word.c_str());
if (rv) {
if ((rv->astr) &&
(TESTAFF(rv->astr, pAMgr->get_forbiddenword(), rv->alen) ||
TESTAFF(rv->astr, pAMgr->get_nosuggest(), rv->alen) ||
TESTAFF(rv->astr, pAMgr->get_substandard(), rv->alen)))
return 0;
while (rv) {
if (rv->astr &&
(TESTAFF(rv->astr, pAMgr->get_needaffix(), rv->alen) ||
TESTAFF(rv->astr, ONLYUPCASEFLAG, rv->alen) ||
TESTAFF(rv->astr, pAMgr->get_onlyincompound(), rv->alen))) {
rv = rv->next_homonym;
} else
break;
}
} else
rv = pAMgr->prefix_check(word.c_str(), word.size(),
0); // only prefix, and prefix + suffix XXX
if (rv) {
nosuffix = 1;
} else {
rv = pAMgr->suffix_check(word.c_str(), word.size(), 0, NULL,
FLAG_NULL, FLAG_NULL, IN_CPD_NOT); // only suffix
}
if (!rv && pAMgr->have_contclass()) {
rv = pAMgr->suffix_check_twosfx(word.c_str(), word.size(), 0, NULL, FLAG_NULL);
if (!rv)
rv = pAMgr->prefix_check_twosfx(word.c_str(), word.size(), 0, FLAG_NULL);
}
// check forbidden words
if ((rv) && (rv->astr) &&
(TESTAFF(rv->astr, pAMgr->get_forbiddenword(), rv->alen) ||
TESTAFF(rv->astr, ONLYUPCASEFLAG, rv->alen) ||
TESTAFF(rv->astr, pAMgr->get_nosuggest(), rv->alen) ||
TESTAFF(rv->astr, pAMgr->get_onlyincompound(), rv->alen)))
return 0;
if (rv) { // XXX obsolote
if ((pAMgr->get_compoundflag()) &&
TESTAFF(rv->astr, pAMgr->get_compoundflag(), rv->alen))
return 2 + nosuffix;
return 1;
}
}
return 0;
}
int SuggestMgr::check_forbidden(const char* word, int len) {
if (pAMgr) {
struct hentry* rv = pAMgr->lookup(word);
if (rv && rv->astr &&
(TESTAFF(rv->astr, pAMgr->get_needaffix(), rv->alen) ||
TESTAFF(rv->astr, pAMgr->get_onlyincompound(), rv->alen)))
rv = NULL;
if (!(pAMgr->prefix_check(word, len, 1)))
rv = pAMgr->suffix_check(word, len, 0, NULL,
FLAG_NULL, FLAG_NULL, IN_CPD_NOT); // prefix+suffix, suffix
// check forbidden words
if ((rv) && (rv->astr) &&
TESTAFF(rv->astr, pAMgr->get_forbiddenword(), rv->alen))
return 1;
}
return 0;
}
std::string SuggestMgr::suggest_morph(const std::string& in_w) {
std::string result;
struct hentry* rv = NULL;
if (!pAMgr)
return std::string();
std::string w(in_w);
// word reversing wrapper for complex prefixes
if (complexprefixes) {
if (utf8)
reverseword_utf(w);
else
reverseword(w);
}
rv = pAMgr->lookup(w.c_str());
while (rv) {
if ((!rv->astr) ||
!(TESTAFF(rv->astr, pAMgr->get_forbiddenword(), rv->alen) ||
TESTAFF(rv->astr, pAMgr->get_needaffix(), rv->alen) ||
TESTAFF(rv->astr, pAMgr->get_onlyincompound(), rv->alen))) {
if (!HENTRY_FIND(rv, MORPH_STEM)) {
result.push_back(MSEP_FLD);
result.append(MORPH_STEM);
result.append(w);
}
if (HENTRY_DATA(rv)) {
result.push_back(MSEP_FLD);
result.append(HENTRY_DATA2(rv));
}
result.push_back(MSEP_REC);
}
rv = rv->next_homonym;
}
std::string st = pAMgr->affix_check_morph(w.c_str(), w.size());
if (!st.empty()) {
result.append(st);
}
if (pAMgr->get_compound() && result.empty()) {
struct hentry* rwords[100]; // buffer for COMPOUND pattern checking
pAMgr->compound_check_morph(w.c_str(), w.size(), 0, 0, 100, 0, NULL, (hentry**)&rwords, 0, result,
NULL);
}
line_uniq(result, MSEP_REC);
return result;
}
static int get_sfxcount(const char* morph) {
if (!morph || !*morph)
return 0;
int n = 0;
const char* old = morph;
morph = strstr(morph, MORPH_DERI_SFX);
if (!morph)
morph = strstr(old, MORPH_INFL_SFX);
if (!morph)
morph = strstr(old, MORPH_TERM_SFX);
while (morph) {
n++;
old = morph;
morph = strstr(morph + 1, MORPH_DERI_SFX);
if (!morph)
morph = strstr(old + 1, MORPH_INFL_SFX);
if (!morph)
morph = strstr(old + 1, MORPH_TERM_SFX);
}
return n;
}
/* affixation */
std::string SuggestMgr::suggest_hentry_gen(hentry* rv, const char* pattern) {
std::string result;
int sfxcount = get_sfxcount(pattern);
if (get_sfxcount(HENTRY_DATA(rv)) > sfxcount)
return result;
if (HENTRY_DATA(rv)) {
std::string aff = pAMgr->morphgen(HENTRY_WORD(rv), rv->blen, rv->astr, rv->alen,
HENTRY_DATA(rv), pattern, 0);
if (!aff.empty()) {
result.append(aff);
result.push_back(MSEP_REC);
}
}
// check all allomorphs
char* p = NULL;
if (HENTRY_DATA(rv))
p = (char*)strstr(HENTRY_DATA2(rv), MORPH_ALLOMORPH);
while (p) {
p += MORPH_TAG_LEN;
int plen = fieldlen(p);
std::string allomorph(p, plen);
struct hentry* rv2 = pAMgr->lookup(allomorph.c_str());
while (rv2) {
// if (HENTRY_DATA(rv2) && get_sfxcount(HENTRY_DATA(rv2)) <=
// sfxcount) {
if (HENTRY_DATA(rv2)) {
char* st = (char*)strstr(HENTRY_DATA2(rv2), MORPH_STEM);
if (st && (strncmp(st + MORPH_TAG_LEN, HENTRY_WORD(rv),
fieldlen(st + MORPH_TAG_LEN)) == 0)) {
std::string aff = pAMgr->morphgen(HENTRY_WORD(rv2), rv2->blen, rv2->astr,
rv2->alen, HENTRY_DATA(rv2), pattern, 0);
if (!aff.empty()) {
result.append(aff);
result.push_back(MSEP_REC);
}
}
}
rv2 = rv2->next_homonym;
}
p = strstr(p + plen, MORPH_ALLOMORPH);
}
return result;
}
std::string SuggestMgr::suggest_gen(const std::vector<std::string>& desc, const std::string& in_pattern) {
if (desc.empty() || !pAMgr)
return std::string();
const char* pattern = in_pattern.c_str();
std::string result2;
std::string newpattern;
struct hentry* rv = NULL;
// search affixed forms with and without derivational suffixes
while (1) {
for (size_t k = 0; k < desc.size(); ++k) {
std::string result;
// add compound word parts (except the last one)
const char* s = desc[k].c_str();
const char* part = strstr(s, MORPH_PART);
if (part) {
const char* nextpart = strstr(part + 1, MORPH_PART);
while (nextpart) {
std::string field;
copy_field(field, part, MORPH_PART);
result.append(field);
part = nextpart;
nextpart = strstr(part + 1, MORPH_PART);
}
s = part;
}
std::string tok(s);
size_t pos = tok.find(" | ");
while (pos != std::string::npos) {
tok[pos + 1] = MSEP_ALT;
pos = tok.find(" | ", pos);
}
std::vector<std::string> pl = line_tok(tok, MSEP_ALT);
for (size_t i = 0; i < pl.size(); ++i) {
// remove inflectional and terminal suffixes
size_t is = pl[i].find(MORPH_INFL_SFX);
if (is != std::string::npos)
pl[i].resize(is);
size_t ts = pl[i].find(MORPH_TERM_SFX);
while (ts != std::string::npos) {
pl[i][ts] = '_';
ts = pl[i].find(MORPH_TERM_SFX);
}
const char* st = strstr(s, MORPH_STEM);
if (st) {
copy_field(tok, st, MORPH_STEM);
rv = pAMgr->lookup(tok.c_str());
while (rv) {
std::string newpat(pl[i]);
newpat.append(pattern);
std::string sg = suggest_hentry_gen(rv, newpat.c_str());
if (sg.empty())
sg = suggest_hentry_gen(rv, pattern);
if (!sg.empty()) {
std::vector<std::string> gen = line_tok(sg, MSEP_REC);
for (size_t j = 0; j < gen.size(); ++j) {
result2.push_back(MSEP_REC);
result2.append(result);
if (pl[i].find(MORPH_SURF_PFX) != std::string::npos) {
std::string field;
copy_field(field, pl[i], MORPH_SURF_PFX);
result2.append(field);
}
result2.append(gen[j]);
}
}
rv = rv->next_homonym;
}
}
}
}
if (!result2.empty() || !strstr(pattern, MORPH_DERI_SFX))
break;
newpattern.assign(pattern);
mystrrep(newpattern, MORPH_DERI_SFX, MORPH_TERM_SFX);
pattern = newpattern.c_str();
}
return result2;
}
// generate an n-gram score comparing s1 and s2, UTF16 version
int SuggestMgr::ngram(int n,
const std::vector<w_char>& su1,
const std::vector<w_char>& su2,
int opt) {
int nscore = 0;
int ns;
int l1;
int l2;
int test = 0;
l1 = su1.size();
l2 = su2.size();
if (l2 == 0)
return 0;
for (int j = 1; j <= n; j++) {
ns = 0;
for (int i = 0; i <= (l1 - j); i++) {
int k = 0;
for (int l = 0; l <= (l2 - j); l++) {
for (k = 0; k < j; k++) {
const w_char& c1 = su1[i + k];
const w_char& c2 = su2[l + k];
if ((c1.l != c2.l) || (c1.h != c2.h))
break;
}
if (k == j) {
ns++;
break;
}
}
if (k != j && opt & NGRAM_WEIGHTED) {
ns--;
test++;
if (i == 0 || i == l1 - j)
ns--; // side weight
}
}
nscore = nscore + ns;
if (ns < 2 && !(opt & NGRAM_WEIGHTED))
break;
}
ns = 0;
if (opt & NGRAM_LONGER_WORSE)
ns = (l2 - l1) - 2;
if (opt & NGRAM_ANY_MISMATCH)
ns = abs(l2 - l1) - 2;
ns = (nscore - ((ns > 0) ? ns : 0));
return ns;
}
// generate an n-gram score comparing s1 and s2, non-UTF16 version
int SuggestMgr::ngram(int n,
const std::string& s1,
const std::string& s2,
int opt) {
int nscore = 0;
int ns;
int l1;
int l2;
int test = 0;
l2 = s2.size();
if (l2 == 0)
return 0;
l1 = s1.size();
for (int j = 1; j <= n; j++) {
ns = 0;
for (int i = 0; i <= (l1 - j); i++) {
//s2 is haystack, s1[i..i+j) is needle
if (s2.find(s1.c_str()+i, 0, j) != std::string::npos) {
ns++;
} else if (opt & NGRAM_WEIGHTED) {
ns--;
test++;
if (i == 0 || i == l1 - j)
ns--; // side weight
}
}
nscore = nscore + ns;
if (ns < 2 && !(opt & NGRAM_WEIGHTED))
break;
}
ns = 0;
if (opt & NGRAM_LONGER_WORSE)
ns = (l2 - l1) - 2;
if (opt & NGRAM_ANY_MISMATCH)
ns = abs(l2 - l1) - 2;
ns = (nscore - ((ns > 0) ? ns : 0));
return ns;
}
// length of the left common substring of s1 and (decapitalised) s2, UTF version
int SuggestMgr::leftcommonsubstring(
const std::vector<w_char>& su1,
const std::vector<w_char>& su2) {
int l1 = su1.size();
int l2 = su2.size();
// decapitalize dictionary word
if (complexprefixes) {
if (su1[l1 - 1] == su2[l2 - 1])
return 1;
} else {
unsigned short idx = su2.empty() ? 0 : (su2[0].h << 8) + su2[0].l;
unsigned short otheridx = su1.empty() ? 0 : (su1[0].h << 8) + su1[0].l;
if (otheridx != idx && (otheridx != unicodetolower(idx, langnum)))
return 0;
int i;
for (i = 1; (i < l1) && (i < l2) && (su1[i].l == su2[i].l) &&
(su1[i].h == su2[i].h);
i++)
;
return i;
}
return 0;
}
// length of the left common substring of s1 and (decapitalised) s2, non-UTF
int SuggestMgr::leftcommonsubstring(
const char* s1,
const char* s2) {
if (complexprefixes) {
int l1 = strlen(s1);
int l2 = strlen(s2);
if (l1 <= l2 && s2[l1 - 1] == s2[l2 - 1])
return 1;
} else if (csconv) {
const char* olds = s1;
// decapitalise dictionary word
if ((*s1 != *s2) && (*s1 != csconv[((unsigned char)*s2)].clower))
return 0;
do {
s1++;
s2++;
} while ((*s1 == *s2) && (*s1 != '\0'));
return (int)(s1 - olds);
}
return 0;
}
int SuggestMgr::commoncharacterpositions(const char* s1,
const char* s2,
int* is_swap) {
int num = 0;
int diff = 0;
int diffpos[2];
*is_swap = 0;
if (utf8) {
std::vector<w_char> su1;
std::vector<w_char> su2;
int l1 = u8_u16(su1, s1);
int l2 = u8_u16(su2, s2);
if (l1 <= 0 || l2 <= 0)
return 0;
// decapitalize dictionary word
if (complexprefixes) {
su2[l2 - 1] = lower_utf(su2[l2 - 1], langnum);
} else {
su2[0] = lower_utf(su2[0], langnum);
}
for (int i = 0; (i < l1) && (i < l2); i++) {
if (su1[i] == su2[i]) {
num++;
} else {
if (diff < 2)
diffpos[diff] = i;
diff++;
}
}
if ((diff == 2) && (l1 == l2) &&
(su1[diffpos[0]] == su2[diffpos[1]]) &&
(su1[diffpos[1]] == su2[diffpos[0]]))
*is_swap = 1;
} else {
size_t i;
std::string t(s2);
// decapitalize dictionary word
if (complexprefixes) {
size_t l2 = t.size();
t[l2 - 1] = csconv[(unsigned char)t[l2 - 1]].clower;
} else {
mkallsmall(t, csconv);
}
for (i = 0; i < t.size() && (*(s1 + i) != 0); ++i) {
if (*(s1 + i) == t[i]) {
num++;
} else {
if (diff < 2)
diffpos[diff] = i;
diff++;
}
}
if ((diff == 2) && (*(s1 + i) == 0) && i == t.size() &&
(*(s1 + diffpos[0]) == t[diffpos[1]]) &&
(*(s1 + diffpos[1]) == t[diffpos[0]]))
*is_swap = 1;
}
return num;
}
int SuggestMgr::mystrlen(const char* word) {
if (utf8) {
std::vector<w_char> w;
return u8_u16(w, word);
} else
return strlen(word);
}
// sort in decreasing order of score
void SuggestMgr::bubblesort(char** rword, char** rword2, int* rsc, int n) {
int m = 1;
while (m < n) {
int j = m;
while (j > 0) {
if (rsc[j - 1] < rsc[j]) {
int sctmp = rsc[j - 1];
char* wdtmp = rword[j - 1];
rsc[j - 1] = rsc[j];
rword[j - 1] = rword[j];
rsc[j] = sctmp;
rword[j] = wdtmp;
if (rword2) {
wdtmp = rword2[j - 1];
rword2[j - 1] = rword2[j];
rword2[j] = wdtmp;
}
j--;
} else
break;
}
m++;
}
return;
}
// longest common subsequence
void SuggestMgr::lcs(const char* s,
const char* s2,
int* l1,
int* l2,
char** result) {
int n, m;
std::vector<w_char> su;
std::vector<w_char> su2;
char* b;
char* c;
int i;
int j;
if (utf8) {
m = u8_u16(su, s);
n = u8_u16(su2, s2);
} else {
m = strlen(s);
n = strlen(s2);
}
c = (char*)malloc((m + 1) * (n + 1));
b = (char*)malloc((m + 1) * (n + 1));
if (!c || !b) {
if (c)
free(c);
if (b)
free(b);
*result = NULL;
return;
}
for (i = 1; i <= m; i++)
c[i * (n + 1)] = 0;
for (j = 0; j <= n; j++)
c[j] = 0;
for (i = 1; i <= m; i++) {
for (j = 1; j <= n; j++) {
if (((utf8) && (su[i - 1] == su2[j - 1])) ||
((!utf8) && (s[i - 1] == s2[j - 1]))) {
c[i * (n + 1) + j] = c[(i - 1) * (n + 1) + j - 1] + 1;
b[i * (n + 1) + j] = LCS_UPLEFT;
} else if (c[(i - 1) * (n + 1) + j] >= c[i * (n + 1) + j - 1]) {
c[i * (n + 1) + j] = c[(i - 1) * (n + 1) + j];
b[i * (n + 1) + j] = LCS_UP;
} else {
c[i * (n + 1) + j] = c[i * (n + 1) + j - 1];
b[i * (n + 1) + j] = LCS_LEFT;
}
}
}
*result = b;
free(c);
*l1 = m;
*l2 = n;
}
int SuggestMgr::lcslen(const char* s, const char* s2) {
int m;
int n;
int i;
int j;
char* result;
int len = 0;
lcs(s, s2, &m, &n, &result);
if (!result)
return 0;
i = m;
j = n;
while ((i != 0) && (j != 0)) {
if (result[i * (n + 1) + j] == LCS_UPLEFT) {
len++;
i--;
j--;
} else if (result[i * (n + 1) + j] == LCS_UP) {
i--;
} else
j--;
}
free(result);
return len;
}
int SuggestMgr::lcslen(const std::string& s, const std::string& s2) {
return lcslen(s.c_str(), s2.c_str());
}
libs/hunspell-1.7.0/src/hunspell/suggestmgr.hxx 0 → 100644
Просмотр файла @ 5ab5623b
/* ***** BEGIN LICENSE BLOCK *****
* Version: MPL 1.1/GPL 2.0/LGPL 2.1
*
* Copyright (C) 2002-2017 Németh László
*
* The contents of this file are subject to the Mozilla Public License Version
* 1.1 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* http://www.mozilla.org/MPL/
*
* Software distributed under the License is distributed on an "AS IS" basis,
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
* for the specific language governing rights and limitations under the
* License.
*
* Hunspell is based on MySpell which is Copyright (C) 2002 Kevin Hendricks.
*
* Contributor(s): David Einstein, Davide Prina, Giuseppe Modugno,
* Gianluca Turconi, Simon Brouwer, Noll János, Bíró Árpád,
* Goldman Eleonóra, Sarlós Tamás, Bencsáth Boldizsár, Halácsy Péter,
* Dvornik László, Gefferth András, Nagy Viktor, Varga Dániel, Chris Halls,
* Rene Engelhard, Bram Moolenaar, Dafydd Jones, Harri Pitkänen
*
* Alternatively, the contents of this file may be used under the terms of
* either the GNU General Public License Version 2 or later (the "GPL"), or
* the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
* in which case the provisions of the GPL or the LGPL are applicable instead
* of those above. If you wish to allow use of your version of this file only
* under the terms of either the GPL or the LGPL, and not to allow others to
* use your version of this file under the terms of the MPL, indicate your
* decision by deleting the provisions above and replace them with the notice
* and other provisions required by the GPL or the LGPL. If you do not delete
* the provisions above, a recipient may use your version of this file under
* the terms of any one of the MPL, the GPL or the LGPL.
*
* ***** END LICENSE BLOCK ***** */
/*
* Copyright 2002 Kevin B. Hendricks, Stratford, Ontario, Canada
* And Contributors. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* 3. All modifications to the source code must be clearly marked as
* such. Binary redistributions based on modified source code
* must be clearly marked as modified versions in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY KEVIN B. HENDRICKS AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
* KEVIN B. HENDRICKS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifndef SUGGESTMGR_HXX_
#define SUGGESTMGR_HXX_
#define MAX_ROOTS 100
#define MAX_WORDS 100
#define MAX_GUESS 200
#define MAXNGRAMSUGS 4
#define MAXPHONSUGS 2
#define MAXCOMPOUNDSUGS 3
#define NGRAM_LONGER_WORSE (1 << 0)
#define NGRAM_ANY_MISMATCH (1 << 1)
#define NGRAM_LOWERING (1 << 2)
#define NGRAM_WEIGHTED (1 << 3)
#include "atypes.hxx"
#include "affixmgr.hxx"
#include "hashmgr.hxx"
#include "langnum.hxx"
enum { LCS_UP, LCS_LEFT, LCS_UPLEFT };
class SuggestMgr {
private:
SuggestMgr(const SuggestMgr&);
SuggestMgr& operator=(const SuggestMgr&);
private:
char* ckey;
size_t ckeyl;
std::vector<w_char> ckey_utf;
char* ctry;
size_t ctryl;
std::vector<w_char> ctry_utf;
bool lang_with_dash_usage;
AffixMgr* pAMgr;
unsigned int maxSug;
struct cs_info* csconv;
int utf8;
int langnum;
int nosplitsugs;
int maxngramsugs;
int maxcpdsugs;
int complexprefixes;
public:
SuggestMgr(const char* tryme, unsigned int maxn, AffixMgr* aptr);
~SuggestMgr();
bool suggest(std::vector<std::string>& slst, const char* word, int* onlycmpdsug);
void ngsuggest(std::vector<std::string>& slst, const char* word, const std::vector<HashMgr*>& rHMgr, int captype);
std::string suggest_morph(const std::string& word);
std::string suggest_gen(const std::vector<std::string>& pl, const std::string& pattern);
private:
void testsug(std::vector<std::string>& wlst,
const std::string& candidate,
int cpdsuggest,
int* timer,
clock_t* timelimit);
int checkword(const std::string& word, int, int*, clock_t*);
int check_forbidden(const char*, int);
void capchars(std::vector<std::string>&, const char*, int);
int replchars(std::vector<std::string>&, const char*, int);
int doubletwochars(std::vector<std::string>&, const char*, int);
int forgotchar(std::vector<std::string>&, const char*, int);
int swapchar(std::vector<std::string>&, const char*, int);
int longswapchar(std::vector<std::string>&, const char*, int);
int movechar(std::vector<std::string>&, const char*, int);
int extrachar(std::vector<std::string>&, const char*, int);
int badcharkey(std::vector<std::string>&, const char*, int);
int badchar(std::vector<std::string>&, const char*, int);
bool twowords(std::vector<std::string>&, const char*, int, bool);
void capchars_utf(std::vector<std::string>&, const w_char*, int wl, int);
int doubletwochars_utf(std::vector<std::string>&, const w_char*, int wl, int);
int forgotchar_utf(std::vector<std::string>&, const w_char*, int wl, int);
int extrachar_utf(std::vector<std::string>&, const w_char*, int wl, int);
int badcharkey_utf(std::vector<std::string>&, const w_char*, int wl, int);
int badchar_utf(std::vector<std::string>&, const w_char*, int wl, int);
int swapchar_utf(std::vector<std::string>&, const w_char*, int wl, int);
int longswapchar_utf(std::vector<std::string>&, const w_char*, int, int);
int movechar_utf(std::vector<std::string>&, const w_char*, int, int);
int mapchars(std::vector<std::string>&, const char*, int);
int map_related(const char*,
std::string&,
int,
std::vector<std::string>& wlst,
int,
const std::vector<mapentry>&,
int*,
clock_t*);
int ngram(int n, const std::vector<w_char>& su1,
const std::vector<w_char>& su2, int opt);
int ngram(int n, const std::string& s1, const std::string& s2, int opt);
int mystrlen(const char* word);
int leftcommonsubstring(const std::vector<w_char>& su1,
const std::vector<w_char>& su2);
int leftcommonsubstring(const char* s1, const char* s2);
int commoncharacterpositions(const char* s1, const char* s2, int* is_swap);
void bubblesort(char** rwd, char** rwd2, int* rsc, int n);
void lcs(const char* s, const char* s2, int* l1, int* l2, char** result);
int lcslen(const char* s, const char* s2);
int lcslen(const std::string& s, const std::string& s2);
std::string suggest_hentry_gen(hentry* rv, const char* pattern);
};
#endif
libs/hunspell-1.7.0/src/hunspell/utf_info.hxx 0 → 100644
Просмотр файла @ 5ab5623b
Этот отличия не может быть отображен по слишком большой. Вместо этого вы можете использовать просмотреть бинарные данные
libs/hunspell-1.7.0/src/hunspell/w_char.hxx 0 → 100644
Просмотр файла @ 5ab5623b
/* ***** BEGIN LICENSE BLOCK *****
* Version: MPL 1.1/GPL 2.0/LGPL 2.1
*
* Copyright (C) 2002-2017 Németh László
*
* The contents of this file are subject to the Mozilla Public License Version
* 1.1 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* http://www.mozilla.org/MPL/
*
* Software distributed under the License is distributed on an "AS IS" basis,
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
* for the specific language governing rights and limitations under the
* License.
*
* Hunspell is based on MySpell which is Copyright (C) 2002 Kevin Hendricks.
*
* Contributor(s): David Einstein, Davide Prina, Giuseppe Modugno,
* Gianluca Turconi, Simon Brouwer, Noll János, Bíró Árpád,
* Goldman Eleonóra, Sarlós Tamás, Bencsáth Boldizsár, Halácsy Péter,
* Dvornik László, Gefferth András, Nagy Viktor, Varga Dániel, Chris Halls,
* Rene Engelhard, Bram Moolenaar, Dafydd Jones, Harri Pitkänen
*
* Alternatively, the contents of this file may be used under the terms of
* either the GNU General Public License Version 2 or later (the "GPL"), or
* the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
* in which case the provisions of the GPL or the LGPL are applicable instead
* of those above. If you wish to allow use of your version of this file only
* under the terms of either the GPL or the LGPL, and not to allow others to
* use your version of this file under the terms of the MPL, indicate your
* decision by deleting the provisions above and replace them with the notice
* and other provisions required by the GPL or the LGPL. If you do not delete
* the provisions above, a recipient may use your version of this file under
* the terms of any one of the MPL, the GPL or the LGPL.
*
* ***** END LICENSE BLOCK ***** */
#ifndef W_CHAR_HXX_
#define W_CHAR_HXX_
#include <string>
#ifndef GCC
struct w_char {
#else
struct __attribute__((packed)) w_char {
#endif
unsigned char l;
unsigned char h;
friend bool operator<(const w_char a, const w_char b) {
unsigned short a_idx = (a.h << 8) + a.l;
unsigned short b_idx = (b.h << 8) + b.l;
return a_idx < b_idx;
}
friend bool operator==(const w_char a, const w_char b) {
return (((a).l == (b).l) && ((a).h == (b).h));
}
friend bool operator!=(const w_char a, const w_char b) {
return !(a == b);;
}
};
// two character arrays
struct replentry {
std::string pattern;
std::string outstrings[4]; // med, ini, fin, isol
};
#endif
libs/hunspell-1.7.0/src/parsers/Makefile.am 0 → 100644
Просмотр файла @ 5ab5623b
AM_CPPFLAGS=-I${top_builddir}/src/hunspell
noinst_LIBRARIES=libparsers.a
libparsers_a_SOURCES=firstparser.cxx xmlparser.cxx \
latexparser.cxx manparser.cxx \
textparser.cxx htmlparser.cxx \
odfparser.cxx
noinst_PROGRAMS=testparser
testparser_SOURCES=firstparser.cxx firstparser.hxx xmlparser.cxx \
xmlparser.hxx latexparser.cxx latexparser.hxx \
manparser.cxx manparser.hxx testparser.cxx \
textparser.cxx textparser.hxx htmlparser.cxx \
htmlparser.hxx odfparser.hxx odfparser.cxx
# need mystrdup()
LDADD = ../hunspell/libhunspell-1.7.la
libs/hunspell-1.7.0/src/parsers/firstparser.cxx 0 → 100644
Просмотр файла @ 5ab5623b
/* ***** BEGIN LICENSE BLOCK *****
* Version: MPL 1.1/GPL 2.0/LGPL 2.1
*
* Copyright (C) 2002-2017 Németh László
*
* The contents of this file are subject to the Mozilla Public License Version
* 1.1 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* http://www.mozilla.org/MPL/
*
* Software distributed under the License is distributed on an "AS IS" basis,
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
* for the specific language governing rights and limitations under the
* License.
*
* Hunspell is based on MySpell which is Copyright (C) 2002 Kevin Hendricks.
*
* Contributor(s): David Einstein, Davide Prina, Giuseppe Modugno,
* Gianluca Turconi, Simon Brouwer, Noll János, Bíró Árpád,
* Goldman Eleonóra, Sarlós Tamás, Bencsáth Boldizsár, Halácsy Péter,
* Dvornik László, Gefferth András, Nagy Viktor, Varga Dániel, Chris Halls,
* Rene Engelhard, Bram Moolenaar, Dafydd Jones, Harri Pitkänen
*
* Alternatively, the contents of this file may be used under the terms of
* either the GNU General Public License Version 2 or later (the "GPL"), or
* the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
* in which case the provisions of the GPL or the LGPL are applicable instead
* of those above. If you wish to allow use of your version of this file only
* under the terms of either the GPL or the LGPL, and not to allow others to
* use your version of this file under the terms of the MPL, indicate your
* decision by deleting the provisions above and replace them with the notice
* and other provisions required by the GPL or the LGPL. If you do not delete
* the provisions above, a recipient may use your version of this file under
* the terms of any one of the MPL, the GPL or the LGPL.
*
* ***** END LICENSE BLOCK ***** */
#include <cstdlib>
#include <cstring>
#include <cstdio>
#include <ctype.h>
#include "../hunspell/csutil.hxx"
#include "firstparser.hxx"
#ifndef W32
using namespace std;
#endif
FirstParser::FirstParser(const char* wordchars)
: TextParser(wordchars) {
}
FirstParser::~FirstParser() {}
bool FirstParser::next_token(std::string& t) {
t.clear();
const size_t tabpos = line[actual].find('\t');
if (tabpos != std::string::npos && tabpos > token) {
token = tabpos;
t = line[actual].substr(0, tabpos);
return true;
}
return false;
}
libs/hunspell-1.7.0/src/parsers/firstparser.hxx 0 → 100644
Просмотр файла @ 5ab5623b
/* ***** BEGIN LICENSE BLOCK *****
* Version: MPL 1.1/GPL 2.0/LGPL 2.1
*
* Copyright (C) 2002-2017 Németh László
*
* The contents of this file are subject to the Mozilla Public License Version
* 1.1 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* http://www.mozilla.org/MPL/
*
* Software distributed under the License is distributed on an "AS IS" basis,
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
* for the specific language governing rights and limitations under the
* License.
*
* Hunspell is based on MySpell which is Copyright (C) 2002 Kevin Hendricks.
*
* Contributor(s): David Einstein, Davide Prina, Giuseppe Modugno,
* Gianluca Turconi, Simon Brouwer, Noll János, Bíró Árpád,
* Goldman Eleonóra, Sarlós Tamás, Bencsáth Boldizsár, Halácsy Péter,
* Dvornik László, Gefferth András, Nagy Viktor, Varga Dániel, Chris Halls,
* Rene Engelhard, Bram Moolenaar, Dafydd Jones, Harri Pitkänen
*
* Alternatively, the contents of this file may be used under the terms of
* either the GNU General Public License Version 2 or later (the "GPL"), or
* the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
* in which case the provisions of the GPL or the LGPL are applicable instead
* of those above. If you wish to allow use of your version of this file only
* under the terms of either the GPL or the LGPL, and not to allow others to
* use your version of this file under the terms of the MPL, indicate your
* decision by deleting the provisions above and replace them with the notice
* and other provisions required by the GPL or the LGPL. If you do not delete
* the provisions above, a recipient may use your version of this file under
* the terms of any one of the MPL, the GPL or the LGPL.
*
* ***** END LICENSE BLOCK ***** */
#ifndef FIRSTPARSER_HXX_
#define FIRSTPARSER_HXX_
#include "textparser.hxx"
/*
* Check first word of the input line
*
*/
class FirstParser : public TextParser {
public:
explicit FirstParser(const char* wc);
virtual ~FirstParser();
virtual bool next_token(std::string&);
};
#endif
libs/hunspell-1.7.0/src/parsers/htmlparser.cxx 0 → 100644
Просмотр файла @ 5ab5623b
/* ***** BEGIN LICENSE BLOCK *****
* Version: MPL 1.1/GPL 2.0/LGPL 2.1
*
* Copyright (C) 2002-2017 Németh László
*
* The contents of this file are subject to the Mozilla Public License Version
* 1.1 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* http://www.mozilla.org/MPL/
*
* Software distributed under the License is distributed on an "AS IS" basis,
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
* for the specific language governing rights and limitations under the
* License.
*
* Hunspell is based on MySpell which is Copyright (C) 2002 Kevin Hendricks.
*
* Contributor(s): David Einstein, Davide Prina, Giuseppe Modugno,
* Gianluca Turconi, Simon Brouwer, Noll János, Bíró Árpád,
* Goldman Eleonóra, Sarlós Tamás, Bencsáth Boldizsár, Halácsy Péter,
* Dvornik László, Gefferth András, Nagy Viktor, Varga Dániel, Chris Halls,
* Rene Engelhard, Bram Moolenaar, Dafydd Jones, Harri Pitkänen
*
* Alternatively, the contents of this file may be used under the terms of
* either the GNU General Public License Version 2 or later (the "GPL"), or
* the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
* in which case the provisions of the GPL or the LGPL are applicable instead
* of those above. If you wish to allow use of your version of this file only
* under the terms of either the GPL or the LGPL, and not to allow others to
* use your version of this file under the terms of the MPL, indicate your
* decision by deleting the provisions above and replace them with the notice
* and other provisions required by the GPL or the LGPL. If you do not delete
* the provisions above, a recipient may use your version of this file under
* the terms of any one of the MPL, the GPL or the LGPL.
*
* ***** END LICENSE BLOCK ***** */
#include <cstdlib>
#include <cstring>
#include <cstdio>
#include <ctype.h>
#include "../hunspell/csutil.hxx"
#include "htmlparser.hxx"
#ifndef W32
using namespace std;
#endif
static const char* PATTERN[][2] = {{"<script", "</script>"},
{"<style", "</style>"},
{"<code", "</code>"},
{"<samp", "</samp>"},
{"<kbd", "</kbd>"},
{"<var", "</var>"},
{"<listing", "</listing>"},
{"<address", "</address>"},
{"<pre", "</pre>"},
{"<!--", "-->"},
{"<[cdata[", "]]>"}, // XML comment
{"<", ">"}};
#define PATTERN_LEN (sizeof(PATTERN) / (sizeof(char*) * 2))
static const char* PATTERN2[][2] = {
{"<img", "alt="}, // ALT and TITLE attrib handled spec.
{"<img", "title="},
{"<a ", "title="}};
#define PATTERN_LEN2 (sizeof(PATTERN2) / (sizeof(char*) * 2))
static const char* (*PATTERN3)[2] = NULL;
#define PATTERN_LEN3 0
HTMLParser::HTMLParser(const char* wordchars)
: XMLParser(wordchars) {
}
HTMLParser::HTMLParser(const w_char* wordchars, int len)
: XMLParser(wordchars, len) {
}
bool HTMLParser::next_token(std::string& t) {
return XMLParser::next_token(PATTERN, PATTERN_LEN, PATTERN2, PATTERN_LEN2, PATTERN3, PATTERN_LEN3, t);
}
HTMLParser::~HTMLParser() {}
libs/hunspell-1.7.0/src/parsers/htmlparser.hxx 0 → 100644
Просмотр файла @ 5ab5623b
/* ***** BEGIN LICENSE BLOCK *****
* Version: MPL 1.1/GPL 2.0/LGPL 2.1
*
* Copyright (C) 2002-2017 Németh László
*
* The contents of this file are subject to the Mozilla Public License Version
* 1.1 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* http://www.mozilla.org/MPL/
*
* Software distributed under the License is distributed on an "AS IS" basis,
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
* for the specific language governing rights and limitations under the
* License.
*
* Hunspell is based on MySpell which is Copyright (C) 2002 Kevin Hendricks.
*
* Contributor(s): David Einstein, Davide Prina, Giuseppe Modugno,
* Gianluca Turconi, Simon Brouwer, Noll János, Bíró Árpád,
* Goldman Eleonóra, Sarlós Tamás, Bencsáth Boldizsár, Halácsy Péter,
* Dvornik László, Gefferth András, Nagy Viktor, Varga Dániel, Chris Halls,
* Rene Engelhard, Bram Moolenaar, Dafydd Jones, Harri Pitkänen
*
* Alternatively, the contents of this file may be used under the terms of
* either the GNU General Public License Version 2 or later (the "GPL"), or
* the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
* in which case the provisions of the GPL or the LGPL are applicable instead
* of those above. If you wish to allow use of your version of this file only
* under the terms of either the GPL or the LGPL, and not to allow others to
* use your version of this file under the terms of the MPL, indicate your
* decision by deleting the provisions above and replace them with the notice
* and other provisions required by the GPL or the LGPL. If you do not delete
* the provisions above, a recipient may use your version of this file under
* the terms of any one of the MPL, the GPL or the LGPL.
*
* ***** END LICENSE BLOCK ***** */
#ifndef HTMLPARSER_HXX_
#define HTMLPARSER_HXX_
#include "xmlparser.hxx"
/*
* HTML Parser
*
*/
class HTMLParser : public XMLParser {
public:
explicit HTMLParser(const char* wc);
HTMLParser(const w_char* wordchars, int len);
virtual bool next_token(std::string&);
virtual ~HTMLParser();
};
#endif
libs/hunspell-1.7.0/src/parsers/latexparser.cxx 0 → 100644
Просмотр файла @ 5ab5623b
/* ***** BEGIN LICENSE BLOCK *****
* Version: MPL 1.1/GPL 2.0/LGPL 2.1
*
* Copyright (C) 2002-2017 Németh László
*
* The contents of this file are subject to the Mozilla Public License Version
* 1.1 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* http://www.mozilla.org/MPL/
*
* Software distributed under the License is distributed on an "AS IS" basis,
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
* for the specific language governing rights and limitations under the
* License.
*
* Hunspell is based on MySpell which is Copyright (C) 2002 Kevin Hendricks.
*
* Contributor(s): David Einstein, Davide Prina, Giuseppe Modugno,
* Gianluca Turconi, Simon Brouwer, Noll János, Bíró Árpád,
* Goldman Eleonóra, Sarlós Tamás, Bencsáth Boldizsár, Halácsy Péter,
* Dvornik László, Gefferth András, Nagy Viktor, Varga Dániel, Chris Halls,
* Rene Engelhard, Bram Moolenaar, Dafydd Jones, Harri Pitkänen
*
* Alternatively, the contents of this file may be used under the terms of
* either the GNU General Public License Version 2 or later (the "GPL"), or
* the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
* in which case the provisions of the GPL or the LGPL are applicable instead
* of those above. If you wish to allow use of your version of this file only
* under the terms of either the GPL or the LGPL, and not to allow others to
* use your version of this file under the terms of the MPL, indicate your
* decision by deleting the provisions above and replace them with the notice
* and other provisions required by the GPL or the LGPL. If you do not delete
* the provisions above, a recipient may use your version of this file under
* the terms of any one of the MPL, the GPL or the LGPL.
*
* ***** END LICENSE BLOCK ***** */
#include <cstdlib>
#include <cstring>
#include <cstdio>
#include <ctype.h>
#include "../hunspell/csutil.hxx"
#include "latexparser.hxx"
#ifndef W32
using namespace std;
#endif
#define UTF8_APOS "\xe2\x80\x99"
#define APOSTROPHE "'"
static struct {
const char* pat[2];
int arg;
} PATTERN[] = {{{"\\(", "\\)"}, 0},
{{"$$", "$$"}, 0},
{{"$", "$"}, 0},
{{"\\begin{math}", "\\end{math}"}, 0},
{{"\\[", "\\]"}, 0},
{{"\\begin{displaymath}", "\\end{displaymath}"}, 0},
{{"\\begin{equation}", "\\end{equation}"}, 0},
{{"\\begin{equation*}", "\\end{equation*}"}, 0},
{{"\\cite", NULL}, 1},
{{"\\nocite", NULL}, 1},
{{"\\index", NULL}, 1},
{{"\\label", NULL}, 1},
{{"\\ref", NULL}, 1},
{{"\\pageref", NULL}, 1},
{{"\\autoref", NULL}, 1},
{{"\\parbox", NULL}, 1},
{{"\\begin{verbatim}", "\\end{verbatim}"}, 0},
{{"\\verb+", "+"}, 0},
{{"\\verb|", "|"}, 0},
{{"\\verb#", "#"}, 0},
{{"\\verb*", "*"}, 0},
{{"\\documentstyle", "\\begin{document}"}, 0},
{{"\\documentclass", "\\begin{document}"}, 0},
// { { "\\documentclass", NULL } , 1 },
{{"\\usepackage", NULL}, 1},
{{"\\includeonly", NULL}, 1},
{{"\\include", NULL}, 1},
{{"\\input", NULL}, 1},
{{"\\vspace", NULL}, 1},
{{"\\setlength", NULL}, 2},
{{"\\addtolength", NULL}, 2},
{{"\\settowidth", NULL}, 2},
{{"\\rule", NULL}, 2},
{{"\\hspace", NULL}, 1},
{{"\\vspace", NULL}, 1},
{{"\\\\[", "]"}, 0},
{{"\\pagebreak[", "]"}, 0},
{{"\\nopagebreak[", "]"}, 0},
{{"\\enlargethispage", NULL}, 1},
{{"\\begin{tabular}", NULL}, 1},
{{"\\addcontentsline", NULL}, 2},
{{"\\begin{thebibliography}", NULL}, 1},
{{"\\bibliography", NULL}, 1},
{{"\\bibliographystyle", NULL}, 1},
{{"\\bibitem", NULL}, 1},
{{"\\begin", NULL}, 1},
{{"\\end", NULL}, 1},
{{"\\pagestyle", NULL}, 1},
{{"\\pagenumbering", NULL}, 1},
{{"\\thispagestyle", NULL}, 1},
{{"\\newtheorem", NULL}, 2},
{{"\\newcommand", NULL}, 2},
{{"\\renewcommand", NULL}, 2},
{{"\\setcounter", NULL}, 2},
{{"\\addtocounter", NULL}, 1},
{{"\\stepcounter", NULL}, 1},
{{"\\selectlanguage", NULL}, 1},
{{"\\inputencoding", NULL}, 1},
{{"\\hyphenation", NULL}, 1},
{{"\\definecolor", NULL}, 3},
{{"\\color", NULL}, 1},
{{"\\textcolor", NULL}, 1},
{{"\\pagecolor", NULL}, 1},
{{"\\colorbox", NULL}, 2},
{{"\\fcolorbox", NULL}, 2},
{{"\\declaregraphicsextensions", NULL}, 1},
{{"\\psfig", NULL}, 1},
{{"\\url", NULL}, 1},
{{"\\eqref", NULL}, 1},
{{"\\vskip", NULL}, 1},
{{"\\vglue", NULL}, 1},
{{"\'\'", NULL}, 1}};
#define PATTERN_LEN (sizeof(PATTERN) / sizeof(PATTERN[0]))
LaTeXParser::LaTeXParser(const char* wordchars)
: TextParser(wordchars)
, pattern_num(0), depth(0), arg(0), opt(0) {
}
LaTeXParser::LaTeXParser(const w_char* wordchars, int len)
: TextParser(wordchars, len)
, pattern_num(0), depth(0), arg(0), opt(0) {
}
LaTeXParser::~LaTeXParser() {}
int LaTeXParser::look_pattern(int col) {
for (unsigned int i = 0; i < PATTERN_LEN; i++) {
const char* j = line[actual].c_str() + head;
const char* k = PATTERN[i].pat[col];
if (!k)
continue;
while ((*k != '\0') && (tolower(*j) == *k)) {
j++;
k++;
}
if (*k == '\0')
return i;
}
return -1;
}
/*
* LaTeXParser
*
* state 0: not wordchar
* state 1: wordchar
* state 2: comments
* state 3: commands
* state 4: commands with arguments
* state 5: % comment
*
*/
bool LaTeXParser::next_token(std::string& t) {
t.clear();
int i;
int slash = 0;
int apostrophe;
for (;;) {
// fprintf(stderr,"depth: %d, state: %d, , arg: %d, token:
// %s\n",depth,state,arg,line[actual]+head);
switch (state) {
case 0: // non word chars
if ((pattern_num = look_pattern(0)) != -1) {
if (PATTERN[pattern_num].pat[1]) {
state = 2;
} else {
state = 4;
depth = 0;
arg = 0;
opt = 1;
}
head += strlen(PATTERN[pattern_num].pat[0]) - 1;
} else if (line[actual][head] == '%') {
state = 5;
} else if (is_wordchar(line[actual].c_str() + head)) {
state = 1;
token = head;
} else if (line[actual][head] == '\\') {
if (line[actual][head + 1] == '\\' || // \\ (linebreak)
(line[actual][head + 1] == '$') || // \$ (dollar sign)
(line[actual][head + 1] == '%')) { // \% (percent)
head++;
break;
}
state = 3;
}
break;
case 1: // wordchar
apostrophe = 0;
if ((is_wordchar((char*)APOSTROPHE) ||
(is_utf8() && is_wordchar((char*)UTF8_APOS))) &&
!line[actual].empty() && line[actual][head] == '\'' &&
is_wordchar(line[actual].c_str() + head + 1)) {
head++;
} else if (is_utf8() &&
is_wordchar((char*)APOSTROPHE) && // add Unicode apostrophe
// to the WORDCHARS, if
// needed
strncmp(line[actual].c_str() + head, UTF8_APOS, strlen(UTF8_APOS)) ==
0 &&
is_wordchar(line[actual].c_str() + head + strlen(UTF8_APOS))) {
head += strlen(UTF8_APOS) - 1;
} else if (!is_wordchar(line[actual].c_str() + head) ||
(line[actual][head] == '\'' && line[actual][head + 1] == '\'' &&
++apostrophe)) {
state = 0;
bool ok = alloc_token(token, &head, t);
if (apostrophe)
head += 2;
if (ok)
return true;
}
break;
case 2: // comment, labels, etc
if (((i = look_pattern(1)) != -1) &&
(strcmp(PATTERN[i].pat[1], PATTERN[pattern_num].pat[1]) == 0)) {
state = 0;
head += strlen(PATTERN[pattern_num].pat[1]) - 1;
}
break;
case 3: // command
if ((tolower(line[actual][head]) < 'a') ||
(tolower(line[actual][head]) > 'z')) {
state = 0;
head--;
}
break;
case 4: // command with arguments
if (slash && (line[actual][head] != '\0')) {
slash = 0;
head++;
break;
} else if (line[actual][head] == '\\') {
slash = 1;
} else if ((line[actual][head] == '{') ||
((opt) && (line[actual][head] == '['))) {
depth++;
opt = 0;
} else if (line[actual][head] == '}') {
depth--;
if (depth == 0) {
opt = 1;
arg++;
}
if (((depth == 0) && (arg == PATTERN[pattern_num].arg)) ||
(depth < 0)) {
state = 0; // XXX not handles the last optional arg.
}
} else if (line[actual][head] == ']')
depth--;
} // case
if (next_char(line[actual].c_str(), &head)) {
if (state == 5)
state = 0;
return false;
}
}
}
libs/hunspell-1.7.0/src/parsers/latexparser.hxx 0 → 100644
Просмотр файла @ 5ab5623b
/* ***** BEGIN LICENSE BLOCK *****
* Version: MPL 1.1/GPL 2.0/LGPL 2.1
*
* Copyright (C) 2002-2017 Németh László
*
* The contents of this file are subject to the Mozilla Public License Version
* 1.1 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* http://www.mozilla.org/MPL/
*
* Software distributed under the License is distributed on an "AS IS" basis,
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
* for the specific language governing rights and limitations under the
* License.
*
* Hunspell is based on MySpell which is Copyright (C) 2002 Kevin Hendricks.
*
* Contributor(s): David Einstein, Davide Prina, Giuseppe Modugno,
* Gianluca Turconi, Simon Brouwer, Noll János, Bíró Árpád,
* Goldman Eleonóra, Sarlós Tamás, Bencsáth Boldizsár, Halácsy Péter,
* Dvornik László, Gefferth András, Nagy Viktor, Varga Dániel, Chris Halls,
* Rene Engelhard, Bram Moolenaar, Dafydd Jones, Harri Pitkänen
*
* Alternatively, the contents of this file may be used under the terms of
* either the GNU General Public License Version 2 or later (the "GPL"), or
* the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
* in which case the provisions of the GPL or the LGPL are applicable instead
* of those above. If you wish to allow use of your version of this file only
* under the terms of either the GPL or the LGPL, and not to allow others to
* use your version of this file under the terms of the MPL, indicate your
* decision by deleting the provisions above and replace them with the notice
* and other provisions required by the GPL or the LGPL. If you do not delete
* the provisions above, a recipient may use your version of this file under
* the terms of any one of the MPL, the GPL or the LGPL.
*
* ***** END LICENSE BLOCK ***** */
#ifndef LATEXPARSER_HXX_
#define LATEXPARSER_HXX_
#include "textparser.hxx"
/*
* HTML Parser
*
*/
class LaTeXParser : public TextParser {
int pattern_num; // number of comment
int depth; // depth of blocks
int arg; // arguments's number
int opt; // optional argument attrib.
public:
explicit LaTeXParser(const char* wc);
LaTeXParser(const w_char* wordchars, int len);
virtual ~LaTeXParser();
virtual bool next_token(std::string&);
private:
int look_pattern(int col);
};
#endif
Поддерживает Markdown
0% или .
You are about to add 0 people to the discussion. Proceed with caution.
Сначала завершите редактирование этого сообщения!
Пожалуйста, зарегистрируйтесь или чтобы прокомментировать