Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

EstNLTK analyzer #818

Merged
merged 6 commits into from
Dec 20, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
refactor code to avoid flake8 warning
  • Loading branch information
osma committed Nov 22, 2024
commit 66f577d5983e9fe8c7043d410d9186b6e4a3afef
12 changes: 9 additions & 3 deletions annif/analyzer/estnltk.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,24 +2,30 @@

from __future__ import annotations

import importlib

from . import analyzer


class EstNLTKAnalyzer(analyzer.Analyzer):
name = "estnltk"

@staticmethod
def is_available() -> bool:
# return True iff EstNLTK is installed
return importlib.util.find_spec("estnltk") is not None

def __init__(self, param: str, **kwargs) -> None:
self.param = param
super().__init__(**kwargs)

Check warning on line 20 in annif/analyzer/estnltk.py

View check run for this annotation

Codecov / codecov/patch

annif/analyzer/estnltk.py#L19-L20

Added lines #L19 - L20 were not covered by tests

def tokenize_words(self, text: str, filter: bool = True) -> list[str]:
import estnltk

Check warning on line 23 in annif/analyzer/estnltk.py

View check run for this annotation

Codecov / codecov/patch

annif/analyzer/estnltk.py#L23

Added line #L23 was not covered by tests

txt = estnltk.Text(text.strip())
txt.tag_layer()
lemmas = [
return [

Check warning on line 27 in annif/analyzer/estnltk.py

View check run for this annotation

Codecov / codecov/patch

annif/analyzer/estnltk.py#L25-L27

Added lines #L25 - L27 were not covered by tests
lemma
for lemma in [l[0] for l in txt.lemma]
for lemma in [lemmas[0] for lemmas in txt.lemma]
if (not filter or self.is_valid_token(lemma))
]
return lemmas
6 changes: 5 additions & 1 deletion tests/test_analyzer_estnltk.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,18 +3,22 @@
import pytest

import annif.analyzer
import annif.analyzer.estnltk

estnltk = pytest.importorskip("estnltk")
pytestmark = pytest.mark.skipif(
not annif.analyzer.estnltk.EstNLTKAnalyzer.is_available(),
reason="EstNLTK is required",
)


def test_estnltk_tokenize_words():
analyzer = annif.analyzer.get_analyzer("estnltk")
words = analyzer.tokenize_words(

Check warning on line 16 in tests/test_analyzer_estnltk.py

View check run for this annotation

Codecov / codecov/patch

tests/test_analyzer_estnltk.py#L15-L16

Added lines #L15 - L16 were not covered by tests
"""
Aga kõik juhtus iseenesest. Ka köögis oli kõik endine.
"""
)
assert words == [

Check warning on line 21 in tests/test_analyzer_estnltk.py

View check run for this annotation

Codecov / codecov/patch

tests/test_analyzer_estnltk.py#L21

Added line #L21 was not covered by tests
"aga",
"kõik",
"juhtuma",
Expand All @@ -27,14 +31,14 @@


def test_estnltk_tokenize_words_no_filter():
analyzer = annif.analyzer.get_analyzer("estnltk")
words = analyzer.tokenize_words(

Check warning on line 35 in tests/test_analyzer_estnltk.py

View check run for this annotation

Codecov / codecov/patch

tests/test_analyzer_estnltk.py#L34-L35

Added lines #L34 - L35 were not covered by tests
"""
Aga kõik juhtus iseenesest. Ka köögis oli kõik endine.
""",
filter=False,
)
assert words == [

Check warning on line 41 in tests/test_analyzer_estnltk.py

View check run for this annotation

Codecov / codecov/patch

tests/test_analyzer_estnltk.py#L41

Added line #L41 was not covered by tests
"aga",
"kõik",
"juhtuma",
Expand Down